From 73a4e280c335e8bf17f2d9b4171e3f597cff4f1d Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Thu, 30 Apr 2020 09:45:52 +0100 Subject: [PATCH 01/33] WIP --- src/node/rpc/member_frontend.h | 21 ++++++++++++++++-- start_test_network.sh | 16 +++++++------- tests/submit_recovery_share.sh | 40 ++++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 10 deletions(-) create mode 100755 tests/submit_recovery_share.sh diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 5ae8b5190a8b..367680c6efc4 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -37,6 +37,23 @@ namespace ccf DECLARE_JSON_TYPE(SubmitRecoveryShare) DECLARE_JSON_REQUIRED_FIELDS(SubmitRecoveryShare, recovery_share) + struct GetEncryptedRecoveryShare + { + std::string encrypted_recovery_share; + std::string nonce; + + GetEncryptedRecoveryShare() = default; + + GetEncryptedRecoveryShare(const EncryptedShare& encrypted_share_raw) : + encrypted_recovery_share( + tls::b64_from_raw(encrypted_share_raw.encrypted_share)), + nonce(tls::b64_from_raw(encrypted_share_raw.nonce)) + {} + }; + DECLARE_JSON_TYPE(GetEncryptedRecoveryShare) + DECLARE_JSON_REQUIRED_FIELDS( + GetEncryptedRecoveryShare, encrypted_recovery_share, nonce) + class MemberHandlers : public CommonHandlerRegistry { private: @@ -875,13 +892,13 @@ namespace ccf "Recovery share not found for member {}", args.caller_id)); } - return make_success(enc_s.value()); + return make_success(GetEncryptedRecoveryShare(enc_s.value())); }; install( MemberProcs::GET_ENCRYPTED_RECOVERY_SHARE, json_adapter(get_encrypted_recovery_share), Read) - .set_auto_schema() + .set_auto_schema() .set_http_get_only(); auto submit_recovery_share = [this]( diff --git a/start_test_network.sh b/start_test_network.sh index a09425cfe828..bf39b7592ca8 100755 --- a/start_test_network.sh +++ b/start_test_network.sh @@ -4,17 +4,17 @@ set -e -echo "Setting up Python environment..." -if [ ! -f "env/bin/activate" ] - then - python3.7 -m venv env -fi -source env/bin/activate +# echo "Setting up Python environment..." +# if [ ! -f "env/bin/activate" ] +# then +# python3.7 -m venv env +# fi +# source env/bin/activate PATH_HERE=$(dirname "$(realpath -s "$0")") -pip install -q -U -r "${PATH_HERE}"/tests/requirements.txt -echo "Python environment successfully setup" +# pip install -q -U -r "${PATH_HERE}"/tests/requirements.txt +# echo "Python environment successfully setup" CURL_CLIENT=ON \ python "${PATH_HERE}"/tests/start_network.py \ diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh new file mode 100755 index 000000000000..61eb503daf50 --- /dev/null +++ b/tests/submit_recovery_share.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the Apache 2.0 License. + +set -ex + +# TODO: +# 1. No need to base64 encode the share: hex encoding is fine + + +# 1. Retrieve encrypted share +encrypted_share=`curl -sS https://127.208.217.117:59615/members/getEncryptedRecoveryShare --cacert networkcert.pem --cert member0_cert.pem --key member0_privk.pem | jq -r .encrypted_recovery_share` + +nonce=`curl -sS https://127.208.217.117:59615/members/getEncryptedRecoveryShare --cacert networkcert.pem --cert member0_cert.pem --key member0_privk.pem | jq -r .nonce` + +# 2. Decrypt encrypted share + +# i. Retrieve raw member encryption key +# openssl asn1parse -in member0_enc_priv.pem -i -strparse 12 -out key.raw -noout +# member_raw_private_key=`cat key.raw | od -tx1 -An | tr -d '[\n/ ]' | cut -c 5-` +# echo -n -e "{$member_raw_private_key}" > member0_privk.raw + +# TODO: This does not seem right... +# ii. Retrieve raw network encryption key +openssl asn1parse -in network_enc_pubk.pem -i -strparse 9 -out key2.raw -noout +# network_raw_public_key=`cat key2.raw | od -tx1 -An | tr -d '[\n/ ]'` +# echo -n -e "${network_raw_public_key}" > network_pubk.raw + +# iii. decrypt it +echo "About to attempt decryption..." +echo "Encrypted share: ${encrypted_share}" +echo "Nonce: ${nonce}" +echo "Private key: ${member_raw_private_key}" +echo "Public key: ${network_raw_public_key}" +raw_nonce=`echo ${nonce} | openssl base64 -d` +raw_input=`echo ${encrypted_share} | openssl base64 -d` +echo "${raw_input}" | step crypto nacl box open "${raw_nonce}" key2.raw key.raw -raw + +# 3. Submit encrypted share + From 93d38fcf902539afa6fd3a2eaa53b07f2b0a129f Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Thu, 30 Apr 2020 15:19:29 +0100 Subject: [PATCH 02/33] This works (but it is not possible to check that a recovery share is valid) --- src/node/node_state.h | 32 ++++++++++------- src/node/rpc/member_frontend.h | 65 +++++++++++++++++++++------------- src/node/rpc/node_interface.h | 7 ++-- src/node/share_manager.h | 47 +++++++++++++++++++++--- tests/infra/member.py | 14 +++++--- 5 files changed, 115 insertions(+), 50 deletions(-) diff --git a/src/node/node_state.h b/src/node/node_state.h index 14f8d6f5d880..fb927ff7a9ac 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -1000,29 +1000,35 @@ namespace ccf }); }; - bool split_ledger_secrets(Store::Tx& tx) override + ShareManager& get_share_manager() override { - try - { - share_manager.issue_shares(tx); - } - catch (const std::logic_error& e) - { - LOG_FAIL_FMT("Failed to update recovery shares info: {}", e.what()); - return false; - } - return true; + return share_manager; } + // bool split_ledger_secrets(Store::Tx& tx) override + // { + // try + // { + // share_manager.issue_shares(tx); + // } + // catch (const std::logic_error& e) + // { + // LOG_FAIL_FMT("Failed to update recovery shares info: {}", e.what()); + // return false; + // } + // return true; + // } + bool restore_ledger_secrets( - Store::Tx& tx, const std::vector& shares) override + Store::Tx& tx, + const std::map& submitted_shares) override { try { finish_recovery( tx, share_manager.restore_recovery_shares_info( - tx, shares, recovery_ledger_secrets)); + tx, submitted_shares, recovery_ledger_secrets)); recovery_ledger_secrets.clear(); } diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 367680c6efc4..a4f576f1d437 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -32,7 +32,7 @@ namespace ccf struct SubmitRecoveryShare { - std::vector recovery_share; + std::string recovery_share; }; DECLARE_JSON_TYPE(SubmitRecoveryShare) DECLARE_JSON_REQUIRED_FIELDS(SubmitRecoveryShare, recovery_share) @@ -348,13 +348,20 @@ namespace ccf {"update_recovery_shares", [this]( ObjectId proposal_id, Store::Tx& tx, const nlohmann::json& args) { - const auto shares_updated = node.split_ledger_secrets(tx); - if (!shares_updated) + // const auto shares_updated = node.split_ledger_secrets(tx); + try + { + share_manager.issue_shares(tx); + } + catch (const std::logic_error& e) { LOG_FAIL_FMT( - "Proposal {}: Updating recovery shares failed", proposal_id); + "Proposal {}: Updating recovery shares failed ({})", + proposal_id, + e.what()); + return false; } - return shares_updated; + return true; }}, {"set_recovery_threshold", [this]( @@ -383,7 +390,9 @@ namespace ccf g.set_recovery_threshold(new_recovery_threshold); // Update recovery shares (same number of shares) - return node.split_ledger_secrets(tx); + share_manager.issue_shares(tx); + return true; + // return node.split_ledger_secrets(tx); }}, }; @@ -550,17 +559,18 @@ namespace ccf NetworkTables& network; AbstractNodeState& node; - const lua::TxScriptRunner tsr; + ShareManager& share_manager; // For now, shares are not stored in the KV - std::vector pending_shares; + std::map pending_shares; - static constexpr auto SIZE_NONCE = 16; + const lua::TxScriptRunner tsr; public: MemberHandlers(NetworkTables& network, AbstractNodeState& node) : CommonHandlerRegistry(*network.tables, Tables::MEMBER_CERTS), network(network), node(node), + share_manager(node.get_share_manager()), tsr(network) {} @@ -815,12 +825,13 @@ namespace ccf members->put(args.caller_id, *member); // New active members are allocated a new recovery share - if (!node.split_ledger_secrets(args.tx)) - { - return make_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - "Error splitting ledger secrets"); - } + // if (!share_manager.split_ledger_secrets(args.tx)) + share_manager.issue_shares(args.tx); + // { + // return make_error( + // HTTP_STATUS_INTERNAL_SERVER_ERROR, + // "Error splitting ledger secrets"); + // } } return make_success(true); }; @@ -925,19 +936,28 @@ namespace ccf HTTP_STATUS_FORBIDDEN, "Node is already recovering private ledger"); } + // TODO: Unit test the whole thing here! + if (pending_shares.find(args.caller_id) != pending_shares.end()) + { + return make_error( + HTTP_STATUS_FORBIDDEN, + fmt::format( + "Member {} cannot submit their recovery share twice", + args.caller_id)); + } + const auto in = params.get(); + auto raw_recovery_share = tls::raw_from_b64(in.recovery_share); SecretSharing::Share share; std::copy_n( - in.recovery_share.begin(), + raw_recovery_share.begin(), SecretSharing::SHARE_LENGTH, share.begin()); - pending_shares.emplace_back(share); + pending_shares.emplace(args.caller_id, share); if (pending_shares.size() < g.get_recovery_threshold()) { - // The number of shares required to re-assemble the secret has not - // yet been reached return make_success(false); } @@ -988,12 +1008,7 @@ namespace ccf g.add_consensus(in.consensus_type); - if (!node.split_ledger_secrets(tx)) - { - return make_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - "Error splitting ledger secrets"); - } + share_manager.issue_shares(tx); size_t self = g.add_node({in.node_info_network, in.node_cert, diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index baf77d3ae2db..3022056cbf96 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -3,7 +3,7 @@ #pragma once #include "node/entities.h" -#include "node/secret_share.h" +#include "node/share_manager.h" #include "node_call_types.h" namespace ccf @@ -26,8 +26,9 @@ namespace ccf const std::optional>& filter = std::nullopt) = 0; virtual NodeId get_node_id() const = 0; - virtual bool split_ledger_secrets(Store::Tx& tx) = 0; + virtual ShareManager& get_share_manager() = 0; virtual bool restore_ledger_secrets( - Store::Tx& tx, const std::vector& shares) = 0; + Store::Tx& tx, + const std::map& submitted_shares) = 0; }; } diff --git a/src/node/share_manager.h b/src/node/share_manager.h index 5930b98ec812..b21629cf7b1f 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -121,12 +121,12 @@ namespace ccf for (auto const& [member_id, enc_pub_key] : active_members_info) { auto nonce = tls::create_entropy()->random(crypto::Box::NONCE_SIZE); - auto share_raw = std::vector( + auto raw_share = std::vector( shares[share_index].begin(), shares[share_index].end()); auto enc_pub_key_raw = tls::PublicX25519::parse(tls::Pem(enc_pub_key)); auto encrypted_share = crypto::Box::create( - share_raw, + raw_share, nonce, enc_pub_key_raw, network.encryption_key->private_raw); @@ -205,18 +205,55 @@ namespace ccf tx, new_ledger_secret, network.ledger_secrets->get_latest()); } + // bool check_share_is_valid( + // Store::Tx& tx, const std::vector& raw_share, MemberId member_id) + // { + // auto recovery_shares_info = tx.get_view(network.shares)->get(0); + // if (!recovery_shares_info.has_value()) + // { + // throw std::logic_error( + // "Failed to retrieve current recovery shares info"); + // } + + // auto member_info = tx.get_view(network.members)->get(member_id); + // if (!member_info.has_value()) + // { + // throw std::logic_error(fmt::format( + // "Could not check recovery share for unknown member {}", member_id)); + // } + + // bool share_is_valid = false; + // for (auto const& s : recovery_shares_info.value().encrypted_shares) + // { + // if ( + // s.first == member_id && + // encrypt_share(raw_share, s.second.nonce, member_info->keyshare) == + // s.second.encrypted_share) + // { + // share_is_valid = true; + // } + // } + + // return share_is_valid; + // } + // For now, the shares are passed directly to this function. Shares should // be retrieved from the KV instead. std::vector restore_recovery_shares_info( Store::Tx& tx, - const std::vector& shares, + const std::map& shares, const std::list& encrypted_recovery_secrets) { // First, re-assemble the ledger secret wrapping key from the given // shares. Then, unwrap the latest ledger secret and use it to decrypt the // previous ledger secret and so on. - auto ls_wrapping_key = - LedgerSecretWrappingKey(SecretSharing::combine(shares, shares.size())); + std::vector share_vec; + for (auto const& s : shares) + { + share_vec.emplace_back(s.second); + } + auto ls_wrapping_key = LedgerSecretWrappingKey( + SecretSharing::combine(share_vec, share_vec.size())); auto recovery_shares_info = tx.get_view(network.shares)->get(0); if (!recovery_shares_info.has_value()) diff --git a/tests/infra/member.py b/tests/infra/member.py index d008b568f575..467a00113197 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -8,6 +8,9 @@ import infra.crypto import http import os +import base64 + +from loguru import logger as LOG class NoRecoveryShareFound(Exception): @@ -136,15 +139,18 @@ def get_and_decrypt_recovery_share(self, remote_node): os.path.join(self.common_dir, "network_enc_pubk_orig.pem"), ) - nonce_bytes = bytes(r.result["nonce"]) - encrypted_share_bytes = bytes(r.result["encrypted_share"]) - return ctx.decrypt(encrypted_share_bytes, nonce_bytes) + return base64.b64encode( + ctx.decrypt( + base64.b64decode(r.result["encrypted_recovery_share"]), + base64.b64decode(r.result["nonce"]), + ) + ).decode() def submit_recovery_share(self, remote_node, decrypted_recovery_share): with remote_node.member_client(member_id=self.member_id) as mc: r = mc.rpc( "submitRecoveryShare", - params={"recovery_share": list(decrypted_recovery_share)}, + params={"recovery_share": decrypted_recovery_share}, ) assert r.error is None, f"Error submitting recovery share: {r.error}" return r From c7bc148b5f0d3b01db0f564a5dc36b3d44c6b876 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 5 May 2020 14:55:22 +0100 Subject: [PATCH 03/33] WIP --- getting_started/setup_vm/ccf-dependencies.yml | 6 +- sphinx/source/members/adding_member.rst | 8 +- sphinx/source/operators/start_network.rst | 6 +- sphinx/source/users/index.rst | 2 +- src/node/rpc/member_frontend.h | 13 +- src/tls/25519.h | 2 +- src/tls/base64.h | 8 +- src/utils/recovery_share_enc.cpp | 44 ++++++ tests/code_update.py | 84 +++++----- tests/infra/ccf.py | 6 +- tests/infra/consortium.py | 4 +- tests/infra/member.py | 13 +- tests/keygenerator.sh | 20 +-- tests/scurl.sh | 4 +- tests/submit_recovery_share.sh | 146 +++++++++++++++--- 15 files changed, 260 insertions(+), 106 deletions(-) create mode 100644 src/utils/recovery_share_enc.cpp diff --git a/getting_started/setup_vm/ccf-dependencies.yml b/getting_started/setup_vm/ccf-dependencies.yml index e37590bb1c7f..071909e49591 100644 --- a/getting_started/setup_vm/ccf-dependencies.yml +++ b/getting_started/setup_vm/ccf-dependencies.yml @@ -6,6 +6,6 @@ - import_role: name: openenclave tasks_from: install.yml - - import_role: - name: ccf_dependencies - tasks_from: install.yml \ No newline at end of file + # - import_role: + # name: ccf_dependencies + # tasks_from: install.yml \ No newline at end of file diff --git a/sphinx/source/members/adding_member.rst b/sphinx/source/members/adding_member.rst index 80b1c10efd18..0f79272950ac 100644 --- a/sphinx/source/members/adding_member.rst +++ b/sphinx/source/members/adding_member.rst @@ -14,16 +14,16 @@ The ``keygenerator.sh`` script can be used to generate the member’s certificat .. code-block:: bash - $ keygenerator.sh --name=member_name --gen-enc-key + $ keygenerator.sh --name member_name --gen-enc-key -- Generating identity private key and certificate for participant "member_name"... Identity curve: secp384r1 Identity private key generated at: member_name_privk.pem Identity certificate generated at: member_name_cert.pem (to be registered in CCF) -- Generating encryption key pair for participant "member_name"... - Encryption private key generated at: member_name_enc_priv.pem - Encryption public key generated at: member_name_enc_pub.pem (to be registered in CCF) + Encryption private key generated at: member_name_enc_privk.pem + Encryption public key generated at: member_name_enc_pubk.pem (to be registered in CCF) -The member’s private keys (e.g. ``member_name_privk.pem`` and ``member_name_enc_priv.pem``) should be stored on a trusted device while the certificate (e.g. ``member_name_cert.pem``) and public encryption key (e.g. ``member_name_enc_pub.pem``) should be registered in CCF by members. +The member’s private keys (e.g. ``member_name_privk.pem`` and ``member_name_enc_privk.pem``) should be stored on a trusted device while the certificate (e.g. ``member_name_cert.pem``) and public encryption key (e.g. ``member_name_enc_pubk.pem``) should be registered in CCF by members. .. note:: See :ref:`developers/cryptography:Algorithms and Curves` for the list of supported cryptographic curves for member identity. diff --git a/sphinx/source/operators/start_network.rst b/sphinx/source/operators/start_network.rst index b466847ba7f5..367cd102f090 100644 --- a/sphinx/source/operators/start_network.rst +++ b/sphinx/source/operators/start_network.rst @@ -36,7 +36,7 @@ When starting up, the node generates its own key pair and outputs the certificat .. note:: The network certificate should be distributed to users and members to be used as the certificate authority (CA) when establishing a TLS connection with any of the nodes part of the CCF network. When using curl, this is passed as the ``--cacert`` argument. -The certificates and recovery public keys of initial members of the consortium are specified via ``--member-info``. For example, if 3 members should be added to CCF, operators should specify ``--member-info member1_cert.pem,member1_enc_pub.pem``, ``--member-info member2_cert.pem,member2_enc_pub.pem``, ``--member-info member3_cert.pem,member3_enc_pub.pem``. +The certificates and recovery public keys of initial members of the consortium are specified via ``--member-info``. For example, if 3 members should be added to CCF, operators should specify ``--member-info member1_cert.pem,member1_enc_pubk.pem``, ``--member-info member2_cert.pem,member2_enc_pubk.pem``, ``--member-info member3_cert.pem,member3_enc_pubk.pem``. The :term:`constitution`, as defined by the initial members, should be passed via the ``--gov-script`` option. @@ -88,7 +88,7 @@ Using a Configuration File [] network-cert-file = - member-info = "," + member-info = "," gov-script = .. code-block:: ini @@ -103,7 +103,7 @@ Using a Configuration File [] network-cert-file = - member-info = "," + member-info = "," gov-script = To pass configuration files, use the ``--config`` option: ``./cchost --config=config.ini``. An error will be generated if the configuration file contains extra fields. Options in the configuration file will be read along with normal command line arguments. Additional information for configuration files in CLI11 can be found `here `_. diff --git a/sphinx/source/users/index.rst b/sphinx/source/users/index.rst index 13f98bce4d02..aa7f3990a925 100644 --- a/sphinx/source/users/index.rst +++ b/sphinx/source/users/index.rst @@ -5,7 +5,7 @@ To generate the certificate and private key of trusted users should be generated .. code-block:: bash - $ CCF/tests/keygenerator.sh --name=user1 + $ CCF/tests/keygenerator.sh --name user1 -- Generating identity private key and certificate for participant "user1"... Identity private key generated at: user1_privk.pem Identity certificate generated at: user1_cert.pem (to be registered in CCF) diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index a4f576f1d437..091f4de76bff 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -936,7 +936,6 @@ namespace ccf HTTP_STATUS_FORBIDDEN, "Node is already recovering private ledger"); } - // TODO: Unit test the whole thing here! if (pending_shares.find(args.caller_id) != pending_shares.end()) { return make_error( @@ -947,6 +946,8 @@ namespace ccf } const auto in = params.get(); + + // TODO: This seems to crash the server when this fails!! auto raw_recovery_share = tls::raw_from_b64(in.recovery_share); SecretSharing::Share share; @@ -958,7 +959,11 @@ namespace ccf pending_shares.emplace(args.caller_id, share); if (pending_shares.size() < g.get_recovery_threshold()) { - return make_success(false); + return make_success(fmt::format( + "Recovery share successfully submitted. {}/{} recovery shares " + "submitted.", + pending_shares.size(), + g.get_recovery_threshold())); } LOG_DEBUG_FMT( @@ -974,13 +979,13 @@ namespace ccf } pending_shares.clear(); - return make_success(true); + return make_success("End of recovery protocol successfully initiated."); }; install( MemberProcs::SUBMIT_RECOVERY_SHARE, json_adapter(submit_recovery_share), Write) - .set_auto_schema(); + .set_auto_schema(); auto create = [this](Store::Tx& tx, nlohmann::json&& params) { LOG_DEBUG_FMT("Processing create RPC"); diff --git a/src/tls/25519.h b/src/tls/25519.h index 2186a1aece90..e76a30fa3bb3 100644 --- a/src/tls/25519.h +++ b/src/tls/25519.h @@ -12,7 +12,7 @@ namespace tls { - // This class parses and writes x25519 PEM keys following openssl + // This class parses and writes x25519 PEM keys following openssl // SubjectPublicKeyInfo DER format, generated by keygenerator.sh (e.g. for // members' public encryption key). Because the mbedtls version shipped with // Open Enclave does not (yet) support x25519 keys, we parse the key manually diff --git a/src/tls/base64.h b/src/tls/base64.h index 68ba6e801c1e..70622229b543 100644 --- a/src/tls/base64.h +++ b/src/tls/base64.h @@ -21,7 +21,7 @@ namespace tls auto rc = mbedtls_base64_decode(nullptr, 0, &len_written, data, size); if (rc != MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL) { - LOG_FAIL_FMT(fmt::format( + throw std::logic_error(fmt::format( "Could not obtain length of decoded base64 buffer: {}", error_string(rc))); } @@ -32,7 +32,7 @@ namespace tls decoded.data(), decoded.size(), &len_written, data, size); if (rc != 0) { - LOG_FAIL_FMT( + throw std::logic_error( fmt::format("Could not decode base64 string: {}", error_string(rc))); } @@ -47,7 +47,7 @@ namespace tls auto rc = mbedtls_base64_encode(nullptr, 0, &len_written, data, size); if (rc != MBEDTLS_ERR_BASE64_BUFFER_TOO_SMALL) { - LOG_FAIL_FMT(fmt::format( + throw std::logic_error(fmt::format( "Could not obtain length required for encoded base64 buffer: {}", error_string(rc))); } @@ -59,7 +59,7 @@ namespace tls mbedtls_base64_encode(dest, b64_string.size(), &len_written, data, size); if (rc != 0) { - LOG_FAIL_FMT( + throw std::logic_error( fmt::format("Could not encode base64 string: {}", error_string(rc))); } diff --git a/src/utils/recovery_share_enc.cpp b/src/utils/recovery_share_enc.cpp new file mode 100644 index 000000000000..1233258d131d --- /dev/null +++ b/src/utils/recovery_share_enc.cpp @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. + +#include "ds/files.h" +#include "tls/base64.h" + +#include +#include + +int main(int argc, char** argv) +{ + CLI::App app{"recovery share enc"}; + + std::string member_privk_file; + app.add_option( + "--member-enc-privk-file", + member_privk_file, + "Member encryption private key file"); + + std::string network_pubk_file; + app.add_option( + "--network-enc-pubk-file", + network_pubk_file, + "Previous network encryption public key file"); + + std::string recovery_share; + app.add_option( + "--recovery_share", recovery_share, "Encrypted recovery share (base64)"); + + std::string nonce; + app.add_option("--nonce", nonce, "Nonce (base64)"); + + CLI11_PARSE(app, argc, argv); + + // TODO: + // 1. Build and install this in cmake + // 2. Crypto box open + // 3. Output base 64 encoded recovery share + + auto raw_recovery_share = tls::raw_from_b64(recovery_share); + auto raw_nonce = tls::raw_from_b64(nonce); + + +} \ No newline at end of file diff --git a/tests/code_update.py b/tests/code_update.py index 824a1e165539..cb2a85c1ccb1 100644 --- a/tests/code_update.py +++ b/tests/code_update.py @@ -54,48 +54,48 @@ def run(args): code_not_found_exception is not None ), f"Adding a node with unsupported code id {new_code_id} should fail" - # Slow quote verification means that any attempt to add a node may cause an election, so confirm primary after adding node - primary, _ = network.find_primary() - - network.consortium.add_new_code(primary, new_code_id) - - new_nodes = set() - old_nodes_count = len(network.nodes) - new_nodes_count = old_nodes_count + 1 - - LOG.info( - f"Adding more new nodes ({new_nodes_count}) than originally existed ({old_nodes_count})" - ) - for _ in range(0, new_nodes_count): - new_node = network.create_and_trust_node( - args.patched_file_name, "localhost", args - ) - assert new_node - new_nodes.add(new_node) - - LOG.info("Stopping all original nodes") - old_nodes = set(network.nodes).difference(new_nodes) - for node in old_nodes: - LOG.debug(f"Stopping old node {node.node_id}") - node.stop() - - sleep_time = ( - args.pbft_view_change_timeout * 2 / 1000 - if args.consensus == "pbft" - else args.raft_election_timeout * 2 / 1000 - ) - LOG.info(f"Waiting {sleep_time}s for a new primary to be elected...") - time.sleep(sleep_time) - - new_primary, _ = network.find_primary() - LOG.info(f"Waited, new_primary is {new_primary.node_id}") - - LOG.info("Adding another node to the network") - new_node = network.create_and_trust_node( - args.patched_file_name, "localhost", args - ) - assert new_node - network.wait_for_node_commit_sync(args.consensus) + # # Slow quote verification means that any attempt to add a node may cause an election, so confirm primary after adding node + # primary, _ = network.find_primary() + + # network.consortium.add_new_code(primary, new_code_id) + + # new_nodes = set() + # old_nodes_count = len(network.nodes) + # new_nodes_count = old_nodes_count + 1 + + # LOG.info( + # f"Adding more new nodes ({new_nodes_count}) than originally existed ({old_nodes_count})" + # ) + # for _ in range(0, new_nodes_count): + # new_node = network.create_and_trust_node( + # args.patched_file_name, "localhost", args + # ) + # assert new_node + # new_nodes.add(new_node) + + # LOG.info("Stopping all original nodes") + # old_nodes = set(network.nodes).difference(new_nodes) + # for node in old_nodes: + # LOG.debug(f"Stopping old node {node.node_id}") + # node.stop() + + # sleep_time = ( + # args.pbft_view_change_timeout * 2 / 1000 + # if args.consensus == "pbft" + # else args.raft_election_timeout * 2 / 1000 + # ) + # LOG.info(f"Waiting {sleep_time}s for a new primary to be elected...") + # time.sleep(sleep_time) + + # new_primary, _ = network.find_primary() + # LOG.info(f"Waited, new_primary is {new_primary.node_id}") + + # LOG.info("Adding another node to the network") + # new_node = network.create_and_trust_node( + # args.patched_file_name, "localhost", args + # ) + # assert new_node + # network.wait_for_node_commit_sync(args.consensus) if __name__ == "__main__": diff --git a/tests/infra/ccf.py b/tests/infra/ccf.py index b2e8212c4739..003d91021dbe 100644 --- a/tests/infra/ccf.py +++ b/tests/infra/ccf.py @@ -364,8 +364,10 @@ def create_and_trust_node(self, lib_name, host, args, target_node=None): def create_user(self, user_id, curve): infra.proc.ccall( self.key_generator, - f"--name=user{user_id}", - f"--curve={curve.name}", + "--name", + f"user{user_id}", + "--curve", + f"{curve.name}", path=self.common_dir, log_output=False, ).check_returncode() diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 657710b9535e..53102416425f 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -50,7 +50,7 @@ def generate_and_propose_new_member(self, remote_node, curve): ) as cert: new_member_cert_pem = [ord(c) for c in cert.read()] with open( - os.path.join(self.common_dir, f"member{new_member_id}_enc_pub.pem") + os.path.join(self.common_dir, f"member{new_member_id}_enc_pubk.pem") ) as keyshare: new_member_keyshare = [ord(k) for k in keyshare.read()] @@ -74,7 +74,7 @@ def generate_and_add_new_member(self, remote_node, curve): def get_members_info(self): members_certs = [f"member{m.member_id}_cert.pem" for m in self.members] - members_enc_pub = [f"member{m.member_id}_enc_pub.pem" for m in self.members] + members_enc_pub = [f"member{m.member_id}_enc_pubk.pem" for m in self.members] return list(zip(members_certs, members_enc_pub)) def get_active_members(self): diff --git a/tests/infra/member.py b/tests/infra/member.py index 467a00113197..9f0ba3e578cd 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -36,8 +36,10 @@ def __init__(self, member_id, curve, key_generator, common_dir): member = f"member{member_id}" infra.proc.ccall( self.key_generator, - f"--name={member}", - f"--curve={curve.name}", + "--name", + f"{member}", + "--curve", + f"{curve.name}", "--gen-enc-key", path=self.common_dir, log_output=False, @@ -126,6 +128,11 @@ def ack(self, remote_node): self.status = MemberStatus.ACTIVE def get_and_decrypt_recovery_share(self, remote_node): + LOG.warning( + f"About to retrieve and decrypt recovery share... {remote_node.host}:{remote_node.rpc_port}" + ) + while True: + pass with remote_node.member_client(member_id=self.member_id) as mc: r = mc.get("getEncryptedRecoveryShare") @@ -135,7 +142,7 @@ def get_and_decrypt_recovery_share(self, remote_node): # For now, members rely on a copy of the original network encryption # public key to decrypt their shares ctx = infra.crypto.CryptoBoxCtx( - os.path.join(self.common_dir, f"member{self.member_id}_enc_priv.pem"), + os.path.join(self.common_dir, f"member{self.member_id}_enc_privk.pem"), os.path.join(self.common_dir, "network_enc_pubk_orig.pem"), ) diff --git a/tests/keygenerator.sh b/tests/keygenerator.sh index ee29318a4f7f..ec9a5332a3c7 100755 --- a/tests/keygenerator.sh +++ b/tests/keygenerator.sh @@ -20,27 +20,27 @@ generate_encryption_key=false function usage() { + echo "Usage:" + echo " $0 --name participant_name [--curve $DEFAULT_CURVE] [--gen-enc-key]" echo "Generates identity private key and self-signed certificates for CCF participants." echo "Optionally generates a x25519 key pair for share encryption (required for consortium members)." - echo "Usage:""" - echo " $0 --name=participant_name [--curve=$DEFAULT_CURVE] [--gen-enc-key]" echo "" echo "Supported curves are: $SUPPORTED_CURVES" } while [ "$1" != "" ]; do - PARAM=${1%=*} - VALUE=${1#*=} - case $PARAM in + case $1 in -h|-\?|--help) usage exit 0 ;; -n|--name) - name="$VALUE" + name="$2" + shift ;; -c|--curve) - curve="$VALUE" + curve="$2" + shift ;; -g|--gen-enc-key) generate_encryption_key=true @@ -53,7 +53,7 @@ done # Validate parameters if [ -z "$name" ]; then - echo "The name of the participant should be specified (e.g. member0 or user1)" + echo "Error: The name of the participant should be specified (e.g. member0 or user1)" exit 1 fi @@ -92,8 +92,8 @@ echo "Identity certificate generated at: $cert (to be registered in CCF)" if "$generate_encryption_key"; then echo "-- Generating encryption key pair for participant \"$name\"..." - enc_priv="$name"_enc_priv.pem - enc_pub="$name"_enc_pub.pem + enc_priv="$name"_enc_privk.pem + enc_pub="$name"_enc_pubk.pem openssl genpkey -out "$enc_priv" -algorithm "$ENCRYPTION_CURVE" openssl pkey -in "$enc_priv" -pubout -out "$enc_pub" diff --git a/tests/scurl.sh b/tests/scurl.sh index e590b18cc2b6..24fd70e2c24d 100755 --- a/tests/scurl.sh +++ b/tests/scurl.sh @@ -25,12 +25,12 @@ for item in "$@" ; do done if [ -z "$request" ]; then - echo "No request found in arguments (-d or --data-binary)" + echo "Error: No request found in arguments (-d or --data-binary)" exit 1 fi if [ -z "$privk" ]; then - echo "No private key found in arguments (--key)" + echo "Error: No private key found in arguments (--key)" exit 1 fi diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh index 61eb503daf50..add21851c85e 100755 --- a/tests/submit_recovery_share.sh +++ b/tests/submit_recovery_share.sh @@ -2,39 +2,135 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the Apache 2.0 License. -set -ex +set -e -# TODO: -# 1. No need to base64 encode the share: hex encoding is fine +function usage() +{ + echo "Usage:""" + echo " $0 --rpc-address=node_rpc_address --member-enc-privk=member_enc_privk.pem --network-enc-pubk=network_enc_pubk.pem [CURL_OPTIONS]" + echo "Retrieves the encrypted recovery share for a given member, decrypts the share and submit the share to initiate the end of the recovery protocol." + echo "Note: Requires step CLI." +} +while [ "$1" != "" ]; do + case $1 in + -h|-\?|--help) + usage + exit 0 + ;; + --rpc-address) + node_rpc_address="$2" + ;; + --member-enc-privk) + member_enc_privk="$2" + ;; + --network-enc-pubk) + network_enc_pubk="$2" + ;; + *) + break + esac + shift + shift +done -# 1. Retrieve encrypted share -encrypted_share=`curl -sS https://127.208.217.117:59615/members/getEncryptedRecoveryShare --cacert networkcert.pem --cert member0_cert.pem --key member0_privk.pem | jq -r .encrypted_recovery_share` +if ! [ -x "$(command -v step)" ]; then + echo "Error: step CLI is not installed on your system or not in your path." + echo "See https://microsoft.github.io/CCF/master/members/accept_recovery.html#submitting-recovery-shares" + exit 1 +fi -nonce=`curl -sS https://127.208.217.117:59615/members/getEncryptedRecoveryShare --cacert networkcert.pem --cert member0_cert.pem --key member0_privk.pem | jq -r .nonce` +if [ -z "$node_rpc_address" ]; then + echo "Error: No node RPC address in arguments (--rpc-address)" + exit 1 +fi -# 2. Decrypt encrypted share +if [ -z "$member_enc_privk" ]; then + echo "Error: No member encryption private key in arguments (--member-enc-privk)" + exit 1 +fi -# i. Retrieve raw member encryption key -# openssl asn1parse -in member0_enc_priv.pem -i -strparse 12 -out key.raw -noout -# member_raw_private_key=`cat key.raw | od -tx1 -An | tr -d '[\n/ ]' | cut -c 5-` -# echo -n -e "{$member_raw_private_key}" > member0_privk.raw +if [ -z "$network_enc_pubk" ]; then + echo "Error: No defunct network encryption public key in arguments (--network-enc-pubk)" + exit 1 +fi -# TODO: This does not seem right... -# ii. Retrieve raw network encryption key -openssl asn1parse -in network_enc_pubk.pem -i -strparse 9 -out key2.raw -noout -# network_raw_public_key=`cat key2.raw | od -tx1 -An | tr -d '[\n/ ]'` -# echo -n -e "${network_raw_public_key}" > network_pubk.raw +# TODO: Check for errors, probably using || -# iii. decrypt it -echo "About to attempt decryption..." -echo "Encrypted share: ${encrypted_share}" +# Retrieve encrypted recovery share and nonce +resp=$(curl -sS https://${node_rpc_address}/members/getEncryptedRecoveryShare ${@}) +encrypted_share="$(echo ${resp} | jq -r .encrypted_recovery_share)" +nonce="$(echo ${resp} | jq -r .nonce)" + +# GOOD +# encrypted_share="gITiie6qa28H7HccwsxfhhFOzK82d2o2dX9iT+ozK4P9PtMrisPehByE1g2bGzh1sCHuUwQYpv3nME/jIuG6qfj2AQDNZMHebA/BW5ZNvc4ZsxLV4uvtS6zkpQ0yj4JOUnmSRqm/5FSG2H/L+8m+wsX8PdMSBJfqsnaFewIsiwhW" + +# BAD +# encrypted_share="AJBL0MSkenXochLnUrW2SQ0Cb8kzapfULv7e4I+CO+tDQwDSxAh+TRN1" + +# # GOOD +# nonce="iCsAvndW5us2wTLZG70khtMrTlhP0OK4" + +# BAD +# nonce="NlkAqQvA2RLG+xWFSrw+JNP3yAy8Vq5Z" + + +echo "Encrypted recovery share: ${encrypted_share}" echo "Nonce: ${nonce}" -echo "Private key: ${member_raw_private_key}" -echo "Public key: ${network_raw_public_key}" -raw_nonce=`echo ${nonce} | openssl base64 -d` -raw_input=`echo ${encrypted_share} | openssl base64 -d` -echo "${raw_input}" | step crypto nacl box open "${raw_nonce}" key2.raw key.raw -raw -# 3. Submit encrypted share +# Parse raw private key from SubjectPublicKeyInfo DER format, as generated by keygenerator.sh --gen-enc-key +der_header_privk_len=14 +openssl asn1parse -in ${member_enc_privk} -strparse ${der_header_privk_len} -out key.raw -noout + +# Parse raw public key generated by network +der_header_pubk_len=9 +openssl asn1parse -in ${network_enc_pubk} -i -strparse ${der_header_pubk_len} -out key2.raw -noout + +# printf "${nonce}" | base64 -d +# exit 0 + + +# set -x + +# Decrypt encrypted share with nonce, member private key and previous network public key + +# decrypted_share=$(echo -n "${encrypted_share}" | openssl base64 -d | step crypto nacl box open "$(echo -n ${nonce} | openssl base64 -d)" key2.raw key.raw -raw | openssl base64 -A) + +# echo ${nonce} | openssl base64 -d | xargs --null echo "${encrypted_share}" | openssl base64 -d | step crypto nacl box open '{}' key2.raw key.raw -raw + +# echo "$(echo -ne ${nonce} | openssl base64 -d)" + +# echo "" + +# Works: +# echo "bGFsYQo=" | openssl base64 -d | xargs -I{} sh -c "echo \"p91/FnEWQWWc1pWUc/dOuOFW6tB602A9\" | openssl base64 -d | step crypto nacl box open {} alice.pub bob.priv -raw" + +# echo "$nonce" | openssl base64 -d | xargs --null -I{} echo $"{encrypted_share}" | openssl base64 -d | step crypto nacl box open {} key2.raw key.raw -raw + + +# nonce="fi2rgnUTTRcvukMaMLYAkG7NScQ5q32e" +# var=$(echo -n "$nonce" | base64 -d | xargs --null -I {} sh -c "echo {} | hd") +# echo "var: $var" + +# # set -x +# decrypted_share1=$(echo -n ${nonce} | base64 -d | xargs --null -I '{}' sh -c "echo "${encrypted_share}" | base64 -d | step crypto nacl box open \"{}\" key2.raw key.raw -raw | openssl base64 -A") +# set +x + +# # echo ${nonce} | openssl base64 -d | xargs -I{} sh -c "echo {}" + +echo "Decrypted share: ${decrypted_share1}" + +# # Orign: +# decrypted_share=$(echo "${encrypted_share}" | openssl base64 -d | step crypto nacl box open "$(echo ${nonce} | openssl base64 -d)" key2.raw key.raw -raw | openssl base64 -A) + +# echo ${decrypted_share} + +# decrypted_share_base64=$(echo ${decrypted_share} | openssl base64 -A) + +# echo "${decrypted_share}" + + +# # Finally. submit encrypted share +# curl https://${node_rpc_address}/members/submitRecoveryShare ${@} -H "Content-Type: application/json" -d '{"recovery_share": "'${decrypted_share}'"}' +# echo "" \ No newline at end of file From 7f6b5948b0a52cb58efe1768d85f991b8f27e4b5 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Mon, 18 May 2020 16:31:38 +0100 Subject: [PATCH 04/33] Add table for submitted shares --- src/node/entities.h | 1 + src/node/genesis_gen.h | 59 +++++++++++++++++++++++- src/node/network_tables.h | 9 ++++ src/node/node_state.h | 14 ++---- src/node/rpc/member_frontend.h | 46 +++++++++--------- src/node/rpc/node_interface.h | 3 +- src/node/rpc/test/member_voting_test.cpp | 12 ++++- src/node/rpc/test/node_stub.h | 18 ++++++-- src/node/secret_share.h | 13 ++++-- src/node/share_manager.h | 18 ++++++-- src/node/submitted_shares.h | 17 +++++++ 11 files changed, 161 insertions(+), 49 deletions(-) create mode 100644 src/node/submitted_shares.h diff --git a/src/node/entities.h b/src/node/entities.h index f906fdb62cbb..f88b4503989c 100644 --- a/src/node/entities.h +++ b/src/node/entities.h @@ -64,6 +64,7 @@ namespace ccf static constexpr auto SHARES = "ccf.shares"; static constexpr auto USER_CODE_IDS = "ccf.users.code_ids"; static constexpr auto CONFIGURATION = "ccf.config"; + static constexpr auto SUBMITTED_SHARES = "ccf.submitted_shares"; }; using StoreSerialiser = kv::KvStoreSerialiser; diff --git a/src/node/genesis_gen.h b/src/node/genesis_gen.h index 1a742eaf7608..0ea368e64942 100644 --- a/src/node/genesis_gen.h +++ b/src/node/genesis_gen.h @@ -292,7 +292,8 @@ namespace ccf bool service_wait_for_shares() { - auto service_view = tx.get_view(tables.service); + auto [service_view, submitted_shares_view] = + tx.get_view(tables.service, tables.submitted_shares); auto active_service = service_view->get(0); if (!active_service.has_value()) { @@ -310,10 +311,66 @@ namespace ccf active_service->status = ServiceStatus::WAITING_FOR_RECOVERY_SHARES; service_view->put(0, active_service.value()); + submitted_shares_view->put(0, {}); return true; } + // TODO: Better error handling so that exception is sent back to client? + std::optional submit_recovery_share( + MemberId member_id, const std::vector& submitted_recovery_share) + { + auto [service_view, submitted_shares_view, config_view] = + tx.get_view(tables.service, tables.submitted_shares, tables.shares); + auto active_service = service_view->get(0); + if (!active_service.has_value()) + { + LOG_FAIL_FMT("Failed to get active service"); + return std::nullopt; + } + + if (active_service->status != ServiceStatus::WAITING_FOR_RECOVERY_SHARES) + { + LOG_FAIL_FMT( + "Could not submit recovery share on current service: status is not " + "WAITING_FOR_RECOVERY_SHARES"); + return std::nullopt; + } + + auto submitted_shares = submitted_shares_view->get(0); + if (!submitted_shares.has_value()) + { + LOG_FAIL_FMT("Failed to get current submitted recovery shares"); + return std::nullopt; + } + + auto submitted_shares_map = submitted_shares.value(); + if (submitted_shares_map.find(member_id) != submitted_shares_map.end()) + { + LOG_FAIL_FMT( + "Member {} cannot submit their recovery shares twice", member_id); + return std::nullopt; + } + + submitted_shares_map[member_id] = submitted_recovery_share; + submitted_shares_view->put(0, submitted_shares_map); + + return submitted_shares_map.size(); + } + + void clear_submitted_recovery_shares() + { + auto submitted_shares_view = tx.get_view(tables.submitted_shares); + auto submitted_shares = submitted_shares_view->get(0); + if (!submitted_shares.has_value()) + { + throw std::logic_error( + "Failed to get current submitted recovery shares"); + } + + submitted_shares_view->put(0, {}); + } + void trust_node(NodeId node_id) { auto nodes_view = tx.get_view(tables.nodes); diff --git a/src/node/network_tables.h b/src/node/network_tables.h index b1f15dbdf141..b9777ac0f5f9 100644 --- a/src/node/network_tables.h +++ b/src/node/network_tables.h @@ -22,6 +22,7 @@ #include "service.h" #include "shares.h" #include "signatures.h" +#include "submitted_shares.h" #include "users.h" #include "values.h" #include "whitelists.h" @@ -50,6 +51,10 @@ namespace ccf GovernanceHistory& governance_history; ClientSignatures& member_client_signatures; Shares& shares; + // The shares are submitted to the public-only network on recovery. As such, + // the table is public but the shares are encrypted manually with the latest + // ledger secret. + SubmittedShares& submitted_shares; Configuration& config; // @@ -115,6 +120,10 @@ namespace ccf tables->create(Tables::MEMBER_CLIENT_SIGNATURES)), shares( tables->create(Tables::SHARES, kv::SecurityDomain::PUBLIC)), + submitted_shares(tables->create( + Tables::SUBMITTED_SHARES, + kv::SecurityDomain::PUBLIC)), // TODO: Submitted shares should not be + // public!!! users(tables->create(Tables::USERS)), config(tables->create( Tables::CONFIGURATION, kv::SecurityDomain::PUBLIC)), diff --git a/src/node/node_state.h b/src/node/node_state.h index 9e8f8a282241..e6f91948e835 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -837,7 +837,7 @@ namespace ccf return g.service_wait_for_shares(); } - bool finish_recovery( + void finish_recovery( Store::Tx& tx, const std::vector& restored_versions) { std::lock_guard guard(lock); @@ -863,7 +863,6 @@ namespace ccf read_ledger_idx(++ledger_idx); sm.advance(State::readingPrivateLedger); - return true; } // @@ -1024,25 +1023,22 @@ namespace ccf return true; } - bool restore_ledger_secrets( - Store::Tx& tx, const std::vector& shares) override + void restore_ledger_secrets(Store::Tx& tx) override { try { finish_recovery( tx, share_manager.restore_recovery_shares_info( - tx, shares, recovery_ledger_secrets)); + tx, recovery_ledger_secrets)); recovery_ledger_secrets.clear(); } catch (const std::logic_error& e) { - LOG_FAIL_FMT("Failed to restore recovery shares info: {}", e.what()); - return false; + throw std::logic_error( + fmt::format("Failed to restore recovery shares info: {}", e.what())); } - - return true; } NodeId get_node_id() const override diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 5ae8b5190a8b..0d655049a31a 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -534,10 +534,6 @@ namespace ccf NetworkTables& network; AbstractNodeState& node; const lua::TxScriptRunner tsr; - // For now, shares are not stored in the KV - std::vector pending_shares; - - static constexpr auto SIZE_NONCE = 16; public: MemberHandlers(NetworkTables& network, AbstractNodeState& node) : @@ -910,33 +906,35 @@ namespace ccf const auto in = params.get(); - SecretSharing::Share share; - std::copy_n( - in.recovery_share.begin(), - SecretSharing::SHARE_LENGTH, - share.begin()); - - pending_shares.emplace_back(share); - if (pending_shares.size() < g.get_recovery_threshold()) + if ( + g.submit_recovery_share(args.caller_id, in.recovery_share).value() < + g.get_recovery_threshold()) { - // The number of shares required to re-assemble the secret has not - // yet been reached + // TODO: Return the number of recovery shares submitted so far + // TODO: Recovery threshold should not be able to be changed if the + // service is waiting for recovery shares + + // The number of shares required to re-assemble the secret has not yet + // been reached return make_success(false); } LOG_DEBUG_FMT( - "Reached secret sharing threshold {}", pending_shares.size()); + "Reached secret sharing threshold {}", g.get_recovery_threshold()); - if (!node.restore_ledger_secrets(args.tx, pending_shares)) - { - pending_shares.clear(); - return make_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - "Failed to combine recovery shares and initiate end of recovery " - "protocol"); - } + node.restore_ledger_secrets(args.tx); + // if (!node.restore_ledger_secrets(args.tx)) + // { + // // TODO: Instead of clearing, keep the shares and try k-of-n + // instead g.clear_submitted_recovery_shares(); + + // return make_error( + // HTTP_STATUS_INTERNAL_SERVER_ERROR, + // "Failed to combine recovery shares and initiate end of recovery " + // "protocol"); + // } - pending_shares.clear(); + g.clear_submitted_recovery_shares(); return make_success(true); }; install( diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index baf77d3ae2db..045d96e77496 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -27,7 +27,6 @@ namespace ccf virtual NodeId get_node_id() const = 0; virtual bool split_ledger_secrets(Store::Tx& tx) = 0; - virtual bool restore_ledger_secrets( - Store::Tx& tx, const std::vector& shares) = 0; + virtual void restore_ledger_secrets(Store::Tx& tx) = 0; }; } diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 1266d283bdbf..200de9521150 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -60,7 +60,17 @@ const auto operator_gov_script_file = template T parse_response_body(const TResponse& r) { - const auto body_j = jsonrpc::unpack(r.body, jsonrpc::Pack::Text); + nlohmann::json body_j; + try + { + body_j = jsonrpc::unpack(r.body, jsonrpc::Pack::Text); + } + catch (const nlohmann::json::parse_error& e) + { + std::cerr << e.what() << std::endl; + std::cerr << std::string(r.body.begin(), r.body.end()) << std::endl; + } + return body_j.get(); } diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h index 5561649c76f4..60a979212051 100644 --- a/src/node/rpc/test/node_stub.h +++ b/src/node/rpc/test/node_stub.h @@ -91,11 +91,23 @@ namespace ccf return true; } - bool restore_ledger_secrets( - Store::Tx& tx, const std::vector& shares) override + void restore_ledger_secrets(Store::Tx& tx) override { + auto submitted_shares = tx.get_view(network->submitted_shares)->get(0); + if (!submitted_shares.has_value()) + { + throw std::logic_error("Could not find submitted shares"); + } + + std::vector shares; + // for (auto const& s : submitted_shares.value()) + // { + // SecretSharing::Share share; + // std::copy_n( + // s.second.begin(), SecretSharing::SHARE_LENGTH, share.begin()); + // shares.emplace_back(share); + // } SecretSharing::combine(shares, shares.size()); - return true; } NodeId get_node_id() const override diff --git a/src/node/secret_share.h b/src/node/secret_share.h index 719098b1f04b..b8ae8ff32267 100644 --- a/src/node/secret_share.h +++ b/src/node/secret_share.h @@ -41,13 +41,14 @@ namespace ccf { if (n == 0 || n > MAX_NUMBER_SHARES) { - throw std::logic_error( - fmt::format("n not in 1-{} range", MAX_NUMBER_SHARES)); + throw std::logic_error(fmt::format( + "Share creation failed: n not in 1-{} range", MAX_NUMBER_SHARES)); } if (k == 0 || k > n) { - throw std::logic_error(fmt::format("k not in 1-n range (n: {})", n)); + throw std::logic_error(fmt::format( + "Share creation failed: k not in 1-n range (k: {}, n: {})", k, n)); } std::vector shares(n); @@ -65,8 +66,10 @@ namespace ccf { if (k == 0 || k > shares.size()) { - throw std::logic_error( - fmt::format("k not in 1-n range (n: {})", shares.size())); + throw std::logic_error(fmt::format( + "Share combination failed: k not in 1-n range (k: {}, n: {})", + k, + shares.size())); } SplitSecret restored_secret; diff --git a/src/node/share_manager.h b/src/node/share_manager.h index 5930b98ec812..79d5645a8beb 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -205,16 +205,26 @@ namespace ccf tx, new_ledger_secret, network.ledger_secrets->get_latest()); } - // For now, the shares are passed directly to this function. Shares should - // be retrieved from the KV instead. std::vector restore_recovery_shares_info( Store::Tx& tx, - const std::vector& shares, const std::list& encrypted_recovery_secrets) { - // First, re-assemble the ledger secret wrapping key from the given + // First, re-assemble the ledger secret wrapping key from the submitted // shares. Then, unwrap the latest ledger secret and use it to decrypt the // previous ledger secret and so on. + std::vector shares; + auto submitted_shares = tx.get_view(network.submitted_shares)->get(0); + + // TODO: Check that optional is valid + + for (auto const& s : submitted_shares.value()) + { + SecretSharing::Share share; + std::copy_n( + s.second.begin(), SecretSharing::SHARE_LENGTH, share.begin()); + shares.emplace_back(share); + } + auto ls_wrapping_key = LedgerSecretWrappingKey(SecretSharing::combine(shares, shares.size())); diff --git a/src/node/submitted_shares.h b/src/node/submitted_shares.h new file mode 100644 index 000000000000..634d487c22a8 --- /dev/null +++ b/src/node/submitted_shares.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the Apache 2.0 License. +#pragma once + +#include "entities.h" + +#include + +namespace ccf +{ + // The key for this table will always be 0 as there can only be one recovery + // happening at any given time + // TODO: Use the member ID for key instead? Probably more efficient? + + using SubmittedShares = + Store::Map>>; +} \ No newline at end of file From 0a7ed52c4b8883bfa1cdd7d87f8025e1a04b7be9 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 19 May 2020 17:12:29 +0100 Subject: [PATCH 05/33] Add e2e test for recoveries --- src/node/rpc/frontend.h | 3 +- tests/election.py | 16 +++----- tests/infra/ccf.py | 33 ++++++++++----- tests/infra/checker.py | 16 ++++---- tests/infra/consortium.py | 30 +++++++------- tests/infra/node.py | 2 + tests/infra/remote.py | 11 ++--- tests/recovery.py | 84 +++++++++++++++++++++++++++++++++++++-- tests/start_network.py | 5 +-- 9 files changed, 139 insertions(+), 61 deletions(-) diff --git a/src/node/rpc/frontend.h b/src/node/rpc/frontend.h index 9203499d27f3..fade1249ce28 100644 --- a/src/node/rpc/frontend.h +++ b/src/node/rpc/frontend.h @@ -116,7 +116,8 @@ namespace ccf } } ctx->set_response_status(HTTP_STATUS_INTERNAL_SERVER_ERROR); - ctx->set_response_body("RPC could not be forwarded to primary."); + ctx->set_response_body( + "RPC could not be forwarded to unknown primary."); return ctx->serialise_response(); } else diff --git a/tests/election.py b/tests/election.py index b2c51325f86b..2424fbead20b 100644 --- a/tests/election.py +++ b/tests/election.py @@ -58,13 +58,6 @@ def run(args): network.start_and_join(args) current_term = None - # Time before an election completes - max_election_duration = ( - args.pbft_view_change_timeout * 2 / 1000 - if args.consensus == "pbft" - else args.raft_election_timeout * 2 / 1000 - ) - # Number of nodes F to stop until network cannot make progress nodes_to_stop = math.ceil(len(hosts) / 2) if args.consensus == "pbft": @@ -104,9 +97,9 @@ def run(args): primary.stop() LOG.debug( - f"Waiting {max_election_duration} for a new primary to be elected..." + f"Waiting {network.election_duration}s for a new primary to be elected..." ) - time.sleep(max_election_duration) + time.sleep(network.election_duration) # More than F nodes have been stopped, trying to commit any message LOG.debug( @@ -120,9 +113,10 @@ def run(args): except infra.ccf.PrimaryNotFound: pass - LOG.info( - "As expected, primary could not be found after election timeout. Test ended successfully." + LOG.success( + f"As expected, primary could not be found after election duration ({network.election_duration}s)." ) + LOG.success("Test ended successfully.") if __name__ == "__main__": diff --git a/tests/infra/ccf.py b/tests/infra/ccf.py index 0768c92f52d2..2a7b28353b81 100644 --- a/tests/infra/ccf.py +++ b/tests/infra/ccf.py @@ -229,6 +229,13 @@ def _start_all_nodes(self, args, recovery=False, ledger_file=None): except Exception: LOG.exception("Failed to start node {}".format(node.node_id)) raise + + self.election_duration = ( + args.pbft_view_change_timeout * 2 / 1000 + if args.consensus == "pbft" + else args.raft_election_timeout * 2 / 1000 + ) + LOG.info("All nodes started") primary, _ = self.find_primary() @@ -302,20 +309,17 @@ def start_and_join(self, args): LOG.success("***** Network is now open *****") def start_in_recovery( - self, args, ledger_file, common_dir=None, defunct_network_enc_pub=None, + self, args, ledger_file, common_dir=None, ): """ - Recovers a CCF network. + Starts a CCF network in recovery mode. :param args: command line arguments to configure the CCF nodes. :param ledger_file: ledger file to recover from. :param common_dir: common directory containing member and user keys and certs. - :param defunct_network_enc_pub: defunct network encryption public key. """ self.common_dir = common_dir or get_common_folder_name( args.workspace, args.label ) - if defunct_network_enc_pub is None: - defunct_network_enc_pub = self.store_current_network_encryption_key() primary = self._start_all_nodes(args, recovery=True, ledger_file=ledger_file) @@ -325,26 +329,35 @@ def start_in_recovery( common_dir, self.key_generator, remote_node=primary ) - self.consortium.check_for_service(primary, status=ServiceStatus.OPENING) - - for node in self.nodes: + for node in self.get_joined_nodes(): self.wait_for_state( node, "partOfPublicNetwork", timeout=args.ledger_recovery_timeout ) self.wait_for_all_nodes_to_catch_up(primary) LOG.success("All nodes joined public network") + def recover(self, args, defunct_network_enc_pub): + """ + Recovers a CCF network previously started in recovery mode. + :param args: command line arguments to configure the CCF nodes. + :param defunct_network_enc_pub: defunct network encryption public key. + """ + # if defunct_network_enc_pub is None: + # defunct_network_enc_pub = self.store_current_network_encryption_key() + + primary, _ = self.find_primary() + self.consortium.check_for_service(primary, status=ServiceStatus.OPENING) self.consortium.wait_for_all_nodes_to_be_trusted(primary, self.nodes) self.consortium.accept_recovery(primary) self.consortium.recover_with_shares(primary, defunct_network_enc_pub) - for node in self.nodes: + for node in self.get_joined_nodes(): self.wait_for_state( node, "partOfNetwork", timeout=args.ledger_recovery_timeout ) self.consortium.check_for_service( - primary, infra.ccf.ServiceStatus.OPEN, pbft_open=(args.consensus == "pbft") + primary, ServiceStatus.OPEN, pbft_open=(args.consensus == "pbft") ) LOG.success("***** Recovered network is now open *****") diff --git a/tests/infra/checker.py b/tests/infra/checker.py index 5d936e6fb323..5825660d51e9 100644 --- a/tests/infra/checker.py +++ b/tests/infra/checker.py @@ -8,7 +8,7 @@ from infra.tx_status import TxStatus -def wait_for_global_commit(node_client, commit_index, term, mksign=False, timeout=3): +def wait_for_global_commit(client, commit_index, term, mksign=False, timeout=3): """ Given a client to a CCF network and a commit_index/term pair, this function waits for this specific commit index to be globally committed by the @@ -21,13 +21,13 @@ def wait_for_global_commit(node_client, commit_index, term, mksign=False, timeou # Forcing a signature accelerates this process for common operations # (e.g. governance proposals) if mksign: - r = node_client.rpc("mkSign") + r = client.rpc("mkSign") if r.error is not None: raise RuntimeError(f"mkSign returned an error: {r.error}") end_time = time.time() + timeout while time.time() < end_time: - r = node_client.get("tx", {"view": term, "seqno": commit_index}) + r = client.get("tx", {"view": term, "seqno": commit_index}) assert ( r.status == http.HTTPStatus.OK ), f"tx request returned HTTP status {r.status}" @@ -44,8 +44,8 @@ def wait_for_global_commit(node_client, commit_index, term, mksign=False, timeou class Checker: - def __init__(self, node_client=None, notification_queue=None): - self.node_client = node_client + def __init__(self, client=None, notification_queue=None): + self.client = client self.notification_queue = notification_queue self.notified_commit = 0 @@ -69,10 +69,8 @@ def __call__(self, rpc_result, result=None, error=None, timeout=2): result, rpc_result.result ) - if self.node_client: - wait_for_global_commit( - self.node_client, rpc_result.commit, rpc_result.term - ) + if self.client: + wait_for_global_commit(self.client, rpc_result.commit, rpc_result.term) if self.notification_queue: end_time = time.time() + timeout diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 407df2f61c87..671d409b4fa6 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -315,22 +315,22 @@ def accept_recovery(self, remote_node): def recover_with_shares(self, remote_node, defunct_network_enc_pubk): submitted_shares_count = 0 - for m in self.get_active_members(): - decrypted_share = m.get_and_decrypt_recovery_share( - remote_node, defunct_network_enc_pubk - ) - r = m.submit_recovery_share(remote_node, decrypted_share) + with remote_node.node_client() as nc: + check_commit = infra.checker.Checker(nc) - submitted_shares_count += 1 - if submitted_shares_count >= self.recovery_threshold: - assert ( - r.result == True - ), "Shares should be combined when all members have submitted their shares" - break - else: - assert ( - r.result == False - ), "Shares should not be combined until all members have submitted their shares" + for m in self.get_active_members(): + decrypted_share = m.get_and_decrypt_recovery_share( + remote_node, defunct_network_enc_pubk + ) + r = m.submit_recovery_share(remote_node, decrypted_share) + check_commit( + r, + result=True + if submitted_shares_count >= self.recovery_threshold + else False, + ) + + submitted_shares_count += 1 def set_recovery_threshold(self, remote_node, recovery_threshold): script = """ diff --git a/tests/infra/node.py b/tests/infra/node.py index 7ab9f94f4d67..d02e2796d08b 100644 --- a/tests/infra/node.py +++ b/tests/infra/node.py @@ -273,9 +273,11 @@ def member_client(self, member_id=0, **kwargs): def suspend(self): self.remote.suspend() + LOG.info(f"Node {self.node_id} suspended...") def resume(self): self.remote.resume() + LOG.info(f"Node {self.node_id} has resumed from suspension.") @contextmanager diff --git a/tests/infra/remote.py b/tests/infra/remote.py index f18d666994b9..5f5f674f5de9 100644 --- a/tests/infra/remote.py +++ b/tests/infra/remote.py @@ -277,16 +277,13 @@ def pid(self): def suspend(self): _, stdout, _ = self.proc_client.exec_command(f"kill -STOP {self.pid()}") - if stdout.channel.recv_exit_status() == 0: - LOG.info(f"Node {self.name} suspended...") - else: - raise RuntimeError(f"Node {self.name} could not be suspended") + if stdout.channel.recv_exit_status() != 0: + raise RuntimeError(f"Remote {self.name} could not be suspended") def resume(self): _, stdout, _ = self.proc_client.exec_command(f"kill -CONT {self.pid()}") if stdout.channel.recv_exit_status() != 0: - raise RuntimeError(f"Could not resume node {self.name} from suspension!") - LOG.info(f"Node {self.name} resuming from suspension...") + raise RuntimeError(f"Could not resume remote {self.name} from suspension!") def stop(self): """ @@ -440,11 +437,9 @@ def start(self): def suspend(self): self.proc.send_signal(signal.SIGSTOP) - LOG.info(f"Node {self.name} suspended...") def resume(self): self.proc.send_signal(signal.SIGCONT) - LOG.info(f"Node {self.name} resuming from suspension...") def stop(self): """ diff --git a/tests/recovery.py b/tests/recovery.py index 400db19f4104..927c6b21c707 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -4,6 +4,7 @@ import infra.ccf import infra.logging_app as app import suite.test_requirements as reqs +import time from loguru import logger as LOG @@ -13,16 +14,88 @@ def test(network, args): primary, _ = network.find_primary() ledger = primary.get_ledger() + defunct_network_enc_pubk = network.store_current_network_encryption_key() recovered_network = infra.ccf.Network( network.hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, network ) recovered_network.start_in_recovery(args, ledger) + recovered_network.recover(args) + return recovered_network + + +@reqs.description("Recovering a network, kill one node while submitting shares") +@reqs.recover(number_txs=2) +def test_share_resilience(network, args): + old_primary, _ = network.find_primary() + ledger = old_primary.get_ledger() + defunct_network_enc_pubk = network.store_current_network_encryption_key() + + recovered_network = infra.ccf.Network( + network.hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, network + ) + recovered_network.start_in_recovery(args, ledger) + primary, _ = recovered_network.find_primary() + recovered_network.consortium.accept_recovery(primary) + + # Submit all required recovery shares minus one. Last recovery share is + # submitted after a new primary is found. + submitted_shares_count = 0 + for m in recovered_network.consortium.get_active_members(): + with primary.node_client() as nc: + if ( + submitted_shares_count + >= recovered_network.consortium.recovery_threshold - 1 + ): + last_member_to_submit = m + break + + check_commit = infra.checker.Checker(nc) + decrypted_share = m.get_and_decrypt_recovery_share( + primary, defunct_network_enc_pubk + ) + check_commit( + m.submit_recovery_share(primary, decrypted_share), result=False + ) + submitted_shares_count += 1 + + # In theory, check_commit should be sufficient to guarantee that the new primary + # will know about all the recovery shares submitted so far. However, because of + # https://github.com/microsoft/CCF/issues/589, we have to wait for all nodes + # to have committed all transactions. + recovered_network.wait_for_all_nodes_to_catch_up(primary) + + # Here, we kill the current primary instead of just suspending it. + # However, because of https://github.com/microsoft/CCF/issues/99#issuecomment-630875387, + # the new primary will most likely be the previous primary, which defies the point of this test. + primary.stop() + LOG.debug( + f"Waiting {recovered_network.election_duration}s for a new primary to be elected..." + ) + time.sleep(recovered_network.election_duration) + new_primary, _ = recovered_network.find_primary() + assert ( + new_primary is not primary + ), f"Primary {primary.node_id} should have changed after election" + + decrypted_share = last_member_to_submit.get_and_decrypt_recovery_share( + new_primary, defunct_network_enc_pubk + ) + last_member_to_submit.submit_recovery_share(new_primary, decrypted_share) + + for node in recovered_network.get_joined_nodes(): + recovered_network.wait_for_state( + node, "partOfNetwork", timeout=args.ledger_recovery_timeout + ) + + recovered_network.consortium.check_for_service( + new_primary, infra.ccf.ServiceStatus.OPEN, + ) return recovered_network def run(args): - hosts = ["localhost", "localhost"] + hosts = ["localhost", "localhost", "localhost"] txs = app.LoggingTxs() @@ -31,11 +104,14 @@ def run(args): ) as network: network.start_and_join(args) - for _ in range(args.recovery): - recovered_network = test(network, args) + for i in range(args.recovery): + # Alternate between recovery with primary change and stable primary-ship + if i % 2 == 0: + recovered_network = test_share_resilience(network, args) + else: + recovered_network = test(network, args) network.stop_all_nodes() network = recovered_network - LOG.success("Recovery complete on all nodes") diff --git a/tests/start_network.py b/tests/start_network.py index 6afe858430d0..a87c37f47824 100644 --- a/tests/start_network.py +++ b/tests/start_network.py @@ -34,9 +34,8 @@ def run(args): f" - Defunct network public encryption key: {args.network_enc_pubk}" ) LOG.info(f" - Common directory: {args.common_dir}") - network.start_in_recovery( - args, args.ledger, args.common_dir, args.network_enc_pubk - ) + network.start_in_recovery(args, args.ledger, args.common_dir) + network.recover(args, args.network_enc_pubk) else: network.start_and_join(args) From d724db6c8ebf1ef325ab41cc1b2993bb84463a57 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 20 May 2020 14:30:46 +0100 Subject: [PATCH 06/33] Encrypt submitted shares --- src/node/genesis_gen.h | 42 ------------ src/node/node_state.h | 11 +++ src/node/rpc/member_frontend.h | 25 +++++-- src/node/rpc/node_interface.h | 7 +- src/node/share_manager.h | 121 ++++++++++++++++++++++++++++----- tests/infra/consortium.py | 2 +- tests/infra/remote.py | 3 - 7 files changed, 143 insertions(+), 68 deletions(-) diff --git a/src/node/genesis_gen.h b/src/node/genesis_gen.h index 0ea368e64942..b4cf9b8ba2e5 100644 --- a/src/node/genesis_gen.h +++ b/src/node/genesis_gen.h @@ -316,48 +316,6 @@ namespace ccf return true; } - // TODO: Better error handling so that exception is sent back to client? - std::optional submit_recovery_share( - MemberId member_id, const std::vector& submitted_recovery_share) - { - auto [service_view, submitted_shares_view, config_view] = - tx.get_view(tables.service, tables.submitted_shares, tables.shares); - auto active_service = service_view->get(0); - if (!active_service.has_value()) - { - LOG_FAIL_FMT("Failed to get active service"); - return std::nullopt; - } - - if (active_service->status != ServiceStatus::WAITING_FOR_RECOVERY_SHARES) - { - LOG_FAIL_FMT( - "Could not submit recovery share on current service: status is not " - "WAITING_FOR_RECOVERY_SHARES"); - return std::nullopt; - } - - auto submitted_shares = submitted_shares_view->get(0); - if (!submitted_shares.has_value()) - { - LOG_FAIL_FMT("Failed to get current submitted recovery shares"); - return std::nullopt; - } - - auto submitted_shares_map = submitted_shares.value(); - if (submitted_shares_map.find(member_id) != submitted_shares_map.end()) - { - LOG_FAIL_FMT( - "Member {} cannot submit their recovery shares twice", member_id); - return std::nullopt; - } - - submitted_shares_map[member_id] = submitted_recovery_share; - submitted_shares_view->put(0, submitted_shares_map); - - return submitted_shares_map.size(); - } - void clear_submitted_recovery_shares() { auto submitted_shares_view = tx.get_view(tables.submitted_shares); diff --git a/src/node/node_state.h b/src/node/node_state.h index e6f91948e835..1aaa4679e54c 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -960,6 +960,10 @@ namespace ccf std::lock_guard guard(lock); sm.expect(State::partOfNetwork); + // TODO: Check that the service is not waiting for recovery shares + // This is important because the submitted shares are encrypted with the + // latest ledger secrets + // Effects of ledger rekey are only observed from the next transaction, // once the local hook on the secrets table has been triggered. @@ -1009,6 +1013,13 @@ namespace ccf }); }; + ShareManager& get_share_manager() override + { + return share_manager; + } + + // TODO: Use share_manager class directly? + // This will break the unit test but it's probably a good thing! bool split_ledger_secrets(Store::Tx& tx) override { try diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 0d655049a31a..66f8a48e4770 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -533,6 +533,7 @@ namespace ccf NetworkTables& network; AbstractNodeState& node; + ShareManager& share_manager; const lua::TxScriptRunner tsr; public: @@ -540,6 +541,7 @@ namespace ccf CommonHandlerRegistry(*network.tables, Tables::MEMBER_CERTS), network(network), node(node), + share_manager(node.get_share_manager()), tsr(network) {} @@ -889,6 +891,8 @@ namespace ccf return make_error(HTTP_STATUS_FORBIDDEN, "Member is not active"); } + LOG_FAIL_FMT("Submitting recovered share, member {}", args.caller_id); + GenesisGenerator g(this->network, args.tx); if ( g.get_service_status() != ServiceStatus::WAITING_FOR_RECOVERY_SHARES) @@ -906,13 +910,26 @@ namespace ccf const auto in = params.get(); - if ( - g.submit_recovery_share(args.caller_id, in.recovery_share).value() < - g.get_recovery_threshold()) + size_t submitted_shares_count = 0; + try + { + submitted_shares_count = share_manager.submit_recovery_share( + args.tx, args.caller_id, in.recovery_share); + } + catch (const std::logic_error& e) + { + return make_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + fmt::format("Could not submit recovery share: {}", e.what())); + } + + if (submitted_shares_count < g.get_recovery_threshold()) { // TODO: Return the number of recovery shares submitted so far // TODO: Recovery threshold should not be able to be changed if the // service is waiting for recovery shares + // TODO: It should not be possible to rekey while the server is + // waiting for recovery shares // The number of shares required to re-assemble the secret has not yet // been reached @@ -934,7 +951,7 @@ namespace ccf // "protocol"); // } - g.clear_submitted_recovery_shares(); + g.clear_submitted_recovery_shares(); // TODO: We shouldn't need this return make_success(true); }; install( diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index 045d96e77496..e1386069b464 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -3,7 +3,8 @@ #pragma once #include "node/entities.h" -#include "node/secret_share.h" +// #include "node/secret_share.h" +#include "node/share_manager.h" #include "node_call_types.h" namespace ccf @@ -26,7 +27,11 @@ namespace ccf const std::optional>& filter = std::nullopt) = 0; virtual NodeId get_node_id() const = 0; + // TODO: Can call share_manager directly virtual bool split_ledger_secrets(Store::Tx& tx) = 0; + + virtual ShareManager& get_share_manager() = 0; + virtual void restore_ledger_secrets(Store::Tx& tx) = 0; }; } diff --git a/src/node/share_manager.h b/src/node/share_manager.h index 79d5645a8beb..ece82a8628fb 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -97,6 +97,13 @@ namespace ccf std::vector encrypted_ledger_secret; }; + // The ShareManager class provides the interface between the ledger secrets, + // the ccf.shares and ccf.submitted_shares KV tables and the rest of the + // service. In particular, it is used to: + // - Issue new recovery shares whenever required (e.g. on startup, rekey and + // membership updates) + // - Re-assemble the ledger secrets on recovery, once a threshold of members + // have successfully submitted their shares class ShareManager { private: @@ -179,6 +186,67 @@ namespace ccf compute_encrypted_shares(tx, ls_wrapping_key)}); } + // TODO: Move this to a different class? + std::vector encrypt_submitted_share( + const std::vector& submitted_share) + { + // Submitted recovery shares are encrypted with the latest ledger secret. + crypto::GcmCipher encrypted_submitted_share(submitted_share.size()); + + auto iv = tls::create_entropy()->random(crypto::GCM_SIZE_IV); + encrypted_submitted_share.hdr.set_iv(iv.data(), iv.size()); + + crypto::KeyAesGcm(network.ledger_secrets->get_latest().master) + .encrypt( + encrypted_submitted_share.hdr.get_iv(), + submitted_share, + nullb, + encrypted_submitted_share.cipher.data(), + encrypted_submitted_share.hdr.tag); + + return encrypted_submitted_share.serialise(); + } + + std::vector decrypt_submitted_share( + const std::vector& encrypted_submitted_share) + { + crypto::GcmCipher encrypted_share; + encrypted_share.deserialise(encrypted_submitted_share); + std::vector decrypted_share(encrypted_share.cipher.size()); + + crypto::KeyAesGcm(network.ledger_secrets->get_latest().master) + .decrypt( + encrypted_share.hdr.get_iv(), + encrypted_share.hdr.tag, + encrypted_share.cipher, + nullb, + decrypted_share.data()); + + return decrypted_share; + } + + LedgerSecretWrappingKey combine_from_submitted_shares(Store::Tx& tx) + { + auto submitted_shares = tx.get_view(network.submitted_shares)->get(0); + if (!submitted_shares.has_value()) + { + throw std::logic_error("Failed to retrieve current submitted shares"); + } + + std::vector shares; + for (auto const& s : submitted_shares.value()) + { + SecretSharing::Share share; + auto decrypted_share = decrypt_submitted_share(s.second); + std::copy_n( + decrypted_share.begin(), SecretSharing::SHARE_LENGTH, share.begin()); + shares.emplace_back(share); + } + + return LedgerSecretWrappingKey( + SecretSharing::combine(shares, shares.size())); + } + public: ShareManager(NetworkState& network_) : network(network_) {} @@ -210,23 +278,10 @@ namespace ccf const std::list& encrypted_recovery_secrets) { // First, re-assemble the ledger secret wrapping key from the submitted - // shares. Then, unwrap the latest ledger secret and use it to decrypt the - // previous ledger secret and so on. - std::vector shares; - auto submitted_shares = tx.get_view(network.submitted_shares)->get(0); - - // TODO: Check that optional is valid - - for (auto const& s : submitted_shares.value()) - { - SecretSharing::Share share; - std::copy_n( - s.second.begin(), SecretSharing::SHARE_LENGTH, share.begin()); - shares.emplace_back(share); - } + // encrypted shares. Then, unwrap the latest ledger secret and use it to + // decrypt the previous ledger secret and so on. - auto ls_wrapping_key = - LedgerSecretWrappingKey(SecretSharing::combine(shares, shares.size())); + auto ls_wrapping_key = combine_from_submitted_shares(tx); auto recovery_shares_info = tx.get_view(network.shares)->get(0); if (!recovery_shares_info.has_value()) @@ -262,7 +317,7 @@ namespace ccf crypto::GcmCipher encrypted_ls; encrypted_ls.deserialise(i->encrypted_ledger_secret); - auto decrypted_ls = std::vector(encrypted_ls.cipher.size()); + std::vector decrypted_ls(encrypted_ls.cipher.size()); if (!crypto::KeyAesGcm(decryption_key) .decrypt( @@ -289,5 +344,37 @@ namespace ccf return restored_versions; } + + size_t submit_recovery_share( + Store::Tx& tx, + MemberId member_id, + const std::vector& submitted_recovery_share) + { + auto [service_view, submitted_shares_view] = + tx.get_view(network.service, network.submitted_shares); + auto active_service = service_view->get(0); + if (!active_service.has_value()) + { + throw std::logic_error("Failed to get active service"); + } + + auto submitted_shares = submitted_shares_view->get(0); + if (!submitted_shares.has_value()) + { + throw std::logic_error( + "Failed to get current submitted recovery shares"); + } + + auto submitted_shares_map = submitted_shares.value(); + submitted_shares_map[member_id] = + encrypt_submitted_share(submitted_recovery_share); + + submitted_shares_view->put(0, submitted_shares_map); + + LOG_FAIL_FMT( + "submitted_shares_map.size() {}", submitted_shares_map.size()); + + return submitted_shares_map.size(); + } }; } \ No newline at end of file diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 671d409b4fa6..52fdb17847d4 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -323,6 +323,7 @@ def recover_with_shares(self, remote_node, defunct_network_enc_pubk): remote_node, defunct_network_enc_pubk ) r = m.submit_recovery_share(remote_node, decrypted_share) + submitted_shares_count += 1 check_commit( r, result=True @@ -330,7 +331,6 @@ def recover_with_shares(self, remote_node, defunct_network_enc_pubk): else False, ) - submitted_shares_count += 1 def set_recovery_threshold(self, remote_node, recovery_threshold): script = """ diff --git a/tests/infra/remote.py b/tests/infra/remote.py index 5f5f674f5de9..65497e81ac89 100644 --- a/tests/infra/remote.py +++ b/tests/infra/remote.py @@ -705,9 +705,6 @@ def set_perf(self): def get_ledger(self): self.remote.get(self.ledger_file_name, self.common_dir) - LOG.success( - f"Retrieve ledger file {self.ledger_file_name} to {self.common_dir}" - ) return os.path.join(self.common_dir, self.ledger_file_name) def ledger_path(self): From c3d463b630f31231bdd49f8b52a36008bf77e4d3 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 20 May 2020 17:21:49 +0100 Subject: [PATCH 07/33] Fix issue with knowing when to recover from after primary changes --- src/node/node_state.h | 78 +++++++++++++++++++--------------- src/node/rpc/node_call_types.h | 1 + src/node/rpc/node_frontend.h | 3 ++ src/node/rpc/node_interface.h | 2 +- src/node/rpc/serialization.h | 3 +- src/node/rpc/test/node_stub.h | 32 ++++++++++---- 6 files changed, 75 insertions(+), 44 deletions(-) diff --git a/src/node/node_state.h b/src/node/node_state.h index 1aaa4679e54c..10e2892b915e 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -453,6 +453,8 @@ namespace ccf if (resp.public_only) { + last_recovered_commit_idx = resp.last_recovered_commit_idx; + setup_recovery_hook(); sm.advance(State::partOfPublicNetwork); } else @@ -736,7 +738,7 @@ namespace ccf // Shares for the new ledger secret can only be issued now, once the // previous ledger secrets have been recovered share_manager.issue_shares_on_recovery( - tx, last_recovered_commit_idx + 1); + tx, last_recovered_commit_idx + 1); // TODO: Shouldn't be this!!! GenesisGenerator g(network, tx); if (!g.open_service()) { @@ -1430,48 +1432,56 @@ namespace ccf }); } + kv::Version get_last_recovered_commit_idx() override + { + // On recovery, only one node recovers the public ledger and is thus aware + // of the version at which the new ledger secret is applicable from. If + // the primary changes while the network is public-only, the new primary + // should also know at which version the new ledger secret is applicable + // from. + std::lock_guard guard(lock); + return last_recovered_commit_idx; + } + void setup_recovery_hook() { + LOG_FAIL_FMT("Setting recovery hook!"); network.shares.set_local_hook( [this]( kv::Version version, const Shares::State& s, const Shares::Write& w) { - if (is_reading_public_ledger()) + for (auto& [k, v] : w) { - for (auto& [k, v] : w) + kv::Version ledger_secret_version; + if (version == 1) { - kv::Version ledger_secret_version; - if (version == 1) - { - // Special case for the genesis transaction, which is applicable - // from the very first transaction - ledger_secret_version = 1; - } - else - { - // If the version is not set (rekeying), use the version - // from the hook plus one. Otherwise (recovery), use the - // version specified. - ledger_secret_version = - v.value.wrapped_latest_ledger_secret.version == - kv::NoVersion ? - (version + 1) : - v.value.wrapped_latest_ledger_secret.version; - } + // Special case for the genesis transaction, which is applicable + // from the very first transaction + ledger_secret_version = 1; + } + else + { + // If the version is not set (rekeying), use the version + // from the hook plus one. Otherwise (recovery), use the + // version specified. + ledger_secret_version = + v.value.wrapped_latest_ledger_secret.version == kv::NoVersion ? + (version + 1) : + v.value.wrapped_latest_ledger_secret.version; + } - // No encrypted ledger secret are stored in the case of a pure - // re-share (i.e. no ledger rekey). - if ( - !v.value.encrypted_previous_ledger_secret.empty() || - ledger_secret_version == 1) - { - LOG_TRACE_FMT( - "Adding one encrypted recovery ledger secret at {}", - ledger_secret_version); + // No encrypted ledger secret are stored in the case of a pure + // re-share (i.e. no ledger rekey). + if ( + !v.value.encrypted_previous_ledger_secret.empty() || + ledger_secret_version == 1) + { + LOG_TRACE_FMT( + "Adding one encrypted recovery ledger secret at {}", + ledger_secret_version); - recovery_ledger_secrets.push_back( - {ledger_secret_version, - v.value.encrypted_previous_ledger_secret}); - } + recovery_ledger_secrets.push_back( + {ledger_secret_version, + v.value.encrypted_previous_ledger_secret}); } } }); diff --git a/src/node/rpc/node_call_types.h b/src/node/rpc/node_call_types.h index 0d63a6487266..985bbb639d9a 100644 --- a/src/node/rpc/node_call_types.h +++ b/src/node/rpc/node_call_types.h @@ -82,6 +82,7 @@ namespace ccf NodeStatus node_status; NodeId node_id; bool public_only; + kv::Version last_recovered_commit_idx; ConsensusType consensus_type = ConsensusType::RAFT; struct NetworkInfo diff --git a/src/node/rpc/node_frontend.h b/src/node/rpc/node_frontend.h index dd59f26ad09b..da3ace690592 100644 --- a/src/node/rpc/node_frontend.h +++ b/src/node/rpc/node_frontend.h @@ -123,6 +123,7 @@ namespace ccf JoinNetworkNodeToNode::Out({node_status, joining_node_id, node.is_part_of_public_network(), + node.get_last_recovered_commit_idx(), this->network.consensus_type, {*this->network.ledger_secrets.get(), *this->network.identity.get(), @@ -201,6 +202,7 @@ namespace ccf {joining_node_status, existing_node_id.value(), node.is_part_of_public_network(), + node.get_last_recovered_commit_idx(), this->network.consensus_type, {*this->network.ledger_secrets.get(), *this->network.identity.get(), @@ -227,6 +229,7 @@ namespace ccf {node_status, existing_node_id.value(), node.is_part_of_public_network(), + node.get_last_recovered_commit_idx(), this->network.consensus_type, {*this->network.ledger_secrets.get(), *this->network.identity.get(), diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index e1386069b464..089d2c9a131c 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -3,7 +3,6 @@ #pragma once #include "node/entities.h" -// #include "node/secret_share.h" #include "node/share_manager.h" #include "node_call_types.h" @@ -27,6 +26,7 @@ namespace ccf const std::optional>& filter = std::nullopt) = 0; virtual NodeId get_node_id() const = 0; + virtual kv::Version get_last_recovered_commit_idx() = 0; // TODO: Can call share_manager directly virtual bool split_ledger_secrets(Store::Tx& tx) = 0; diff --git a/src/node/rpc/serialization.h b/src/node/rpc/serialization.h index 6415d21fd80e..b4c1448d63cd 100644 --- a/src/node/rpc/serialization.h +++ b/src/node/rpc/serialization.h @@ -55,7 +55,8 @@ namespace ccf node_id, public_only, consensus_type) - DECLARE_JSON_OPTIONAL_FIELDS(JoinNetworkNodeToNode::Out, network_info) + DECLARE_JSON_OPTIONAL_FIELDS( + JoinNetworkNodeToNode::Out, last_recovered_commit_idx, network_info) DECLARE_JSON_TYPE(CreateNetworkNodeToNode::In) DECLARE_JSON_REQUIRED_FIELDS( diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h index 60a979212051..7897e703beb6 100644 --- a/src/node/rpc/test/node_stub.h +++ b/src/node/rpc/test/node_stub.h @@ -13,9 +13,14 @@ namespace ccf bool is_public = false; std::shared_ptr network; + NetworkState network_state; + ShareManager share_manager; + public: StubNodeState(std::shared_ptr network_ = nullptr) : - network(network_) + network(network_), + network_state(ConsensusType::RAFT), // TODO: Remove + share_manager(network_state) // TODO: Remove {} bool accept_recovery(Store::Tx& tx) override @@ -100,16 +105,27 @@ namespace ccf } std::vector shares; - // for (auto const& s : submitted_shares.value()) - // { - // SecretSharing::Share share; - // std::copy_n( - // s.second.begin(), SecretSharing::SHARE_LENGTH, share.begin()); - // shares.emplace_back(share); - // } + for (auto const& s : submitted_shares.value()) + { + SecretSharing::Share share; + std::copy_n( + s.second.begin(), SecretSharing::SHARE_LENGTH, share.begin()); + shares.emplace_back(share); + } SecretSharing::combine(shares, shares.size()); } + kv::Version get_last_recovered_commit_idx() override + { + return kv::NoVersion; + } + + // TODO: Super bad hack for now. Please fix this!! + ShareManager& get_share_manager() override + { + return share_manager; + } + NodeId get_node_id() const override { return 0; From 1e221364f5ed7b00ca58202fa477034aca08f108 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 20 May 2020 22:44:19 +0100 Subject: [PATCH 08/33] Some extra testing --- src/node/genesis_gen.h | 27 ++++++++++++++++++++++++++- src/node/node_state.h | 16 +++++++++++++--- src/node/rpc/member_frontend.h | 10 +--------- tests/infra/clients.py | 2 +- tests/infra/consortium.py | 5 +++++ tests/infra/member.py | 3 +++ 6 files changed, 49 insertions(+), 14 deletions(-) diff --git a/src/node/genesis_gen.h b/src/node/genesis_gen.h index b4cf9b8ba2e5..2be333c453cb 100644 --- a/src/node/genesis_gen.h +++ b/src/node/genesis_gen.h @@ -412,10 +412,35 @@ namespace ccf shares_view->put(0, key_share_info); } - void set_recovery_threshold(size_t threshold) + bool set_recovery_threshold(size_t threshold) { auto config_view = tx.get_view(tables.config); + + // While waiting for recovery shares, the recovery threshold cannot be + // modified. Otherwise, the threshold could be passed without triggering + // the end of recovery procedure + if ( + get_service_status().value() == + ServiceStatus::WAITING_FOR_RECOVERY_SHARES) + { + LOG_FAIL_FMT( + "Failed to set recovery threshold: service is currently waiting for " + "recovery shares"); + return false; + } + + auto active_members_count = get_active_members_count(); + if (threshold > active_members_count) + { + LOG_FAIL_FMT( + "Recovery threshold cannot be set to {} as it is greater than " + "the number of active members ({})", + threshold, + active_members_count); + return false; + } config_view->put(0, {threshold}); + return true; } size_t get_recovery_threshold() diff --git a/src/node/node_state.h b/src/node/node_state.h index 10e2892b915e..6be04152c84e 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -962,9 +962,19 @@ namespace ccf std::lock_guard guard(lock); sm.expect(State::partOfNetwork); - // TODO: Check that the service is not waiting for recovery shares - // This is important because the submitted shares are encrypted with the - // latest ledger secrets + // Because submitted recovery shares are encrypted with the latest ledger + // secret, it is not possible to rekey the ledger if the service is in + // that state. + GenesisGenerator g(network, tx); + if ( + g.get_service_status().value() == + ServiceStatus::WAITING_FOR_RECOVERY_SHARES) + { + LOG_FAIL_FMT( + "Ledger could not be rekeyed while the service is waiting for " + "recovery shares"); + return false; + } // Effects of ledger rekey are only observed from the next transaction, // once the local hook on the secrets table has been triggered. diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 66f8a48e4770..add2c895104b 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -353,17 +353,10 @@ namespace ccf return true; } - auto active_members_count = g.get_active_members_count(); - if (new_recovery_threshold > active_members_count) + if (!g.set_recovery_threshold(new_recovery_threshold)) { - LOG_FAIL_FMT( - "Recovery threshold cannot be set to {} as it is greater than " - "the number of active members ({})", - new_recovery_threshold, - active_members_count); return false; } - g.set_recovery_threshold(new_recovery_threshold); // Update recovery shares (same number of shares) return node.split_ledger_secrets(tx); @@ -909,7 +902,6 @@ namespace ccf } const auto in = params.get(); - size_t submitted_shares_count = 0; try { diff --git a/tests/infra/clients.py b/tests/infra/clients.py index 881f42108186..c9ce6a298bda 100644 --- a/tests/infra/clients.py +++ b/tests/infra/clients.py @@ -422,7 +422,7 @@ def _just_rpc(self, method, *args, **kwargs): description = "" if self.description: - description = f" ({self.description})" + ("[signed]" if is_signed else "") + description = f" ({self.description})" + (" [signed]" if is_signed else "") for logger in self.rpc_loggers: logger.log_request(r, self.name, description) diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 52fdb17847d4..aa336135064a 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -331,6 +331,11 @@ def recover_with_shares(self, remote_node, defunct_network_enc_pubk): else False, ) + # TODO: There is a bug here: when removing the following lines, + # when a member is added and then the network crashes, + # the new member cannot decrypt its share + # if submitted_shares_count >= self.recovery_threshold: + # break def set_recovery_threshold(self, remote_node, recovery_threshold): script = """ diff --git a/tests/infra/member.py b/tests/infra/member.py index d26f7fe87631..551c1bab1090 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -9,6 +9,8 @@ import http import os +from loguru import logger as LOG + class NoRecoveryShareFound(Exception): def __init__(self, response): @@ -150,6 +152,7 @@ def get_and_decrypt_recovery_share(self, remote_node, defunct_network_enc_pubk): nonce_bytes = bytes(r.result["nonce"]) encrypted_share_bytes = bytes(r.result["encrypted_share"]) + LOG.error(ctx.decrypt(encrypted_share_bytes, nonce_bytes)) return ctx.decrypt(encrypted_share_bytes, nonce_bytes) def submit_recovery_share(self, remote_node, decrypted_recovery_share): From a90a315dee4b0d2c5d41db696bba3aae49d99e16 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 20 May 2020 22:44:41 +0100 Subject: [PATCH 09/33] Cleanup --- src/node/node_state.h | 18 +-------------- src/node/rpc/member_frontend.h | 41 +++++++++++++++++++++++----------- src/node/rpc/node_interface.h | 2 +- 3 files changed, 30 insertions(+), 31 deletions(-) diff --git a/src/node/node_state.h b/src/node/node_state.h index 6be04152c84e..8bb02a9871b7 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -738,7 +738,7 @@ namespace ccf // Shares for the new ledger secret can only be issued now, once the // previous ledger secrets have been recovered share_manager.issue_shares_on_recovery( - tx, last_recovered_commit_idx + 1); // TODO: Shouldn't be this!!! + tx, last_recovered_commit_idx + 1); GenesisGenerator g(network, tx); if (!g.open_service()) { @@ -1030,22 +1030,6 @@ namespace ccf return share_manager; } - // TODO: Use share_manager class directly? - // This will break the unit test but it's probably a good thing! - bool split_ledger_secrets(Store::Tx& tx) override - { - try - { - share_manager.issue_shares(tx); - } - catch (const std::logic_error& e) - { - LOG_FAIL_FMT("Failed to update recovery shares info: {}", e.what()); - return false; - } - return true; - } - void restore_ledger_secrets(Store::Tx& tx) override { try diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index add2c895104b..d03a59480aaf 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -331,13 +331,19 @@ namespace ccf {"update_recovery_shares", [this]( ObjectId proposal_id, Store::Tx& tx, const nlohmann::json& args) { - const auto shares_updated = node.split_ledger_secrets(tx); - if (!shares_updated) + try + { + share_manager.issue_shares(tx); + } + catch (const std::logic_error& e) { LOG_FAIL_FMT( - "Proposal {}: Updating recovery shares failed", proposal_id); + "Proposal {}: Updating recovery shares failed: {}", + proposal_id, + e.what()); + return false; } - return shares_updated; + return true; }}, {"set_recovery_threshold", [this]( @@ -359,7 +365,17 @@ namespace ccf } // Update recovery shares (same number of shares) - return node.split_ledger_secrets(tx); + try + { + share_manager.issue_shares(tx); + } + catch (const std::logic_error& e) + { + LOG_FAIL_FMT( + "Proposal {}: Setting recovery threshold failed: {}", e.what()); + return false; + } + return true; }}, }; @@ -789,11 +805,15 @@ namespace ccf members->put(args.caller_id, *member); // New active members are allocated a new recovery share - if (!node.split_ledger_secrets(args.tx)) + try + { + share_manager.issue_shares(args.tx); + } + catch (const std::logic_error& e) { return make_error( HTTP_STATUS_INTERNAL_SERVER_ERROR, - "Error splitting ledger secrets"); + fmt::format("Error issuing new recovery shares {}", e.what())); } } return make_success(true); @@ -978,12 +998,7 @@ namespace ccf g.add_consensus(in.consensus_type); - if (!node.split_ledger_secrets(tx)) - { - return make_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - "Error splitting ledger secrets"); - } + share_manager.issue_shares(tx); size_t self = g.add_node({in.node_info_network, in.node_cert, diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index 089d2c9a131c..c3d3a6c2a501 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -28,7 +28,7 @@ namespace ccf virtual kv::Version get_last_recovered_commit_idx() = 0; // TODO: Can call share_manager directly - virtual bool split_ledger_secrets(Store::Tx& tx) = 0; + // virtual bool split_ledger_secrets(Store::Tx& tx) = 0; virtual ShareManager& get_share_manager() = 0; From 8e3db3910f020cb325323e6c79ea5c4532875b97 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Thu, 21 May 2020 13:34:05 +0100 Subject: [PATCH 10/33] More cleanup --- src/node/genesis_gen.h | 13 ---------- src/node/rpc/member_frontend.h | 44 +++++----------------------------- src/node/share_manager.h | 34 ++++++++++++++++++++++++++ tests/infra/consortium.py | 7 ++---- tests/infra/member.py | 1 - tests/recovery.py | 10 ++++---- 6 files changed, 47 insertions(+), 62 deletions(-) diff --git a/src/node/genesis_gen.h b/src/node/genesis_gen.h index 2be333c453cb..d7fb77561dda 100644 --- a/src/node/genesis_gen.h +++ b/src/node/genesis_gen.h @@ -316,19 +316,6 @@ namespace ccf return true; } - void clear_submitted_recovery_shares() - { - auto submitted_shares_view = tx.get_view(tables.submitted_shares); - auto submitted_shares = submitted_shares_view->get(0); - if (!submitted_shares.has_value()) - { - throw std::logic_error( - "Failed to get current submitted recovery shares"); - } - - submitted_shares_view->put(0, {}); - } - void trust_node(NodeId node_id) { auto nodes_view = tx.get_view(tables.nodes); diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index d03a59480aaf..c7741a49e7dd 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -861,24 +861,10 @@ namespace ccf "Only active members are given recovery shares"); } - std::optional enc_s; - auto recovery_shares_info = - args.tx.get_view(this->network.shares)->get(0); - if (!recovery_shares_info.has_value()) - { - return make_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, - "Failed to retrieve current recovery shares info"); - } - for (auto const& s : recovery_shares_info->encrypted_shares) - { - if (s.first == args.caller_id) - { - enc_s = s.second; - } - } + auto encrypted_share = + share_manager.get_encrypted_share(args.tx, args.caller_id); - if (!enc_s.has_value()) + if (!encrypted_share.has_value()) { return make_error( HTTP_STATUS_BAD_REQUEST, @@ -886,7 +872,7 @@ namespace ccf "Recovery share not found for member {}", args.caller_id)); } - return make_success(enc_s.value()); + return make_success(encrypted_share.value()); }; install( MemberProcs::GET_ENCRYPTED_RECOVERY_SHARE, @@ -904,8 +890,6 @@ namespace ccf return make_error(HTTP_STATUS_FORBIDDEN, "Member is not active"); } - LOG_FAIL_FMT("Submitting recovered share, member {}", args.caller_id); - GenesisGenerator g(this->network, args.tx); if ( g.get_service_status() != ServiceStatus::WAITING_FOR_RECOVERY_SHARES) @@ -937,12 +921,6 @@ namespace ccf if (submitted_shares_count < g.get_recovery_threshold()) { - // TODO: Return the number of recovery shares submitted so far - // TODO: Recovery threshold should not be able to be changed if the - // service is waiting for recovery shares - // TODO: It should not be possible to rekey while the server is - // waiting for recovery shares - // The number of shares required to re-assemble the secret has not yet // been reached return make_success(false); @@ -952,18 +930,8 @@ namespace ccf "Reached secret sharing threshold {}", g.get_recovery_threshold()); node.restore_ledger_secrets(args.tx); - // if (!node.restore_ledger_secrets(args.tx)) - // { - // // TODO: Instead of clearing, keep the shares and try k-of-n - // instead g.clear_submitted_recovery_shares(); - - // return make_error( - // HTTP_STATUS_INTERNAL_SERVER_ERROR, - // "Failed to combine recovery shares and initiate end of recovery " - // "protocol"); - // } - - g.clear_submitted_recovery_shares(); // TODO: We shouldn't need this + + share_manager.clear_submitted_recovery_shares(args.tx); return make_success(true); }; install( diff --git a/src/node/share_manager.h b/src/node/share_manager.h index ece82a8628fb..8e2d6d1925e0 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -273,6 +273,27 @@ namespace ccf tx, new_ledger_secret, network.ledger_secrets->get_latest()); } + std::optional get_encrypted_share( + Store::Tx& tx, MemberId member_id) + { + std::optional encrypted_share = std::nullopt; + auto recovery_shares_info = tx.get_view(network.shares)->get(0); + if (!recovery_shares_info.has_value()) + { + throw std::logic_error( + "Failed to retrieve current recovery shares info"); + } + + for (auto const& s : recovery_shares_info->encrypted_shares) + { + if (s.first == member_id) + { + encrypted_share = s.second; + } + } + return encrypted_share; + } + std::vector restore_recovery_shares_info( Store::Tx& tx, const std::list& encrypted_recovery_secrets) @@ -376,5 +397,18 @@ namespace ccf return submitted_shares_map.size(); } + + void clear_submitted_recovery_shares(Store::Tx& tx) + { + auto submitted_shares_view = tx.get_view(network.submitted_shares); + auto submitted_shares = submitted_shares_view->get(0); + if (!submitted_shares.has_value()) + { + throw std::logic_error( + "Failed to get current submitted recovery shares"); + } + + submitted_shares_view->put(0, {}); + } }; } \ No newline at end of file diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index aa336135064a..880ed5972e22 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -331,11 +331,8 @@ def recover_with_shares(self, remote_node, defunct_network_enc_pubk): else False, ) - # TODO: There is a bug here: when removing the following lines, - # when a member is added and then the network crashes, - # the new member cannot decrypt its share - # if submitted_shares_count >= self.recovery_threshold: - # break + if submitted_shares_count >= self.recovery_threshold: + break def set_recovery_threshold(self, remote_node, recovery_threshold): script = """ diff --git a/tests/infra/member.py b/tests/infra/member.py index 551c1bab1090..7bd0fb40763d 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -152,7 +152,6 @@ def get_and_decrypt_recovery_share(self, remote_node, defunct_network_enc_pubk): nonce_bytes = bytes(r.result["nonce"]) encrypted_share_bytes = bytes(r.result["encrypted_share"]) - LOG.error(ctx.decrypt(encrypted_share_bytes, nonce_bytes)) return ctx.decrypt(encrypted_share_bytes, nonce_bytes) def submit_recovery_share(self, remote_node, decrypted_recovery_share): diff --git a/tests/recovery.py b/tests/recovery.py index 927c6b21c707..114c603a7fbb 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -20,7 +20,7 @@ def test(network, args): network.hosts, args.binary_dir, args.debug_nodes, args.perf_nodes, network ) recovered_network.start_in_recovery(args, ledger) - recovered_network.recover(args) + recovered_network.recover(args, defunct_network_enc_pubk) return recovered_network @@ -106,10 +106,10 @@ def run(args): for i in range(args.recovery): # Alternate between recovery with primary change and stable primary-ship - if i % 2 == 0: - recovered_network = test_share_resilience(network, args) - else: - recovered_network = test(network, args) + # if i % 2 == 0: + recovered_network = test_share_resilience(network, args) + # else: + # recovered_network = test(network, args) network.stop_all_nodes() network = recovered_network LOG.success("Recovery complete on all nodes") From 212c1c86e38f4af1b43dde3b095e47f4e2f2cab7 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Thu, 21 May 2020 14:36:02 +0100 Subject: [PATCH 11/33] Change schema of submitted table --- src/node/genesis_gen.h | 4 +- src/node/network_tables.h | 3 -- src/node/node_state.h | 2 +- src/node/share_manager.h | 80 +++++++++++++++++++++---------------- src/node/submitted_shares.h | 11 ++--- 5 files changed, 53 insertions(+), 47 deletions(-) diff --git a/src/node/genesis_gen.h b/src/node/genesis_gen.h index d7fb77561dda..97db84aefdf7 100644 --- a/src/node/genesis_gen.h +++ b/src/node/genesis_gen.h @@ -292,8 +292,7 @@ namespace ccf bool service_wait_for_shares() { - auto [service_view, submitted_shares_view] = - tx.get_view(tables.service, tables.submitted_shares); + auto service_view = tx.get_view(tables.service); auto active_service = service_view->get(0); if (!active_service.has_value()) { @@ -311,7 +310,6 @@ namespace ccf active_service->status = ServiceStatus::WAITING_FOR_RECOVERY_SHARES; service_view->put(0, active_service.value()); - submitted_shares_view->put(0, {}); return true; } diff --git a/src/node/network_tables.h b/src/node/network_tables.h index b9777ac0f5f9..35cc8e9913b2 100644 --- a/src/node/network_tables.h +++ b/src/node/network_tables.h @@ -51,9 +51,6 @@ namespace ccf GovernanceHistory& governance_history; ClientSignatures& member_client_signatures; Shares& shares; - // The shares are submitted to the public-only network on recovery. As such, - // the table is public but the shares are encrypted manually with the latest - // ledger secret. SubmittedShares& submitted_shares; Configuration& config; diff --git a/src/node/node_state.h b/src/node/node_state.h index 8bb02a9871b7..0c15e82261e5 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -836,6 +836,7 @@ namespace ccf sm.expect(State::partOfPublicNetwork); GenesisGenerator g(network, tx); + share_manager.clear_submitted_recovery_shares(tx); return g.service_wait_for_shares(); } @@ -1439,7 +1440,6 @@ namespace ccf void setup_recovery_hook() { - LOG_FAIL_FMT("Setting recovery hook!"); network.shares.set_local_hook( [this]( kv::Version version, const Shares::State& s, const Shares::Write& w) { diff --git a/src/node/share_manager.h b/src/node/share_manager.h index 8e2d6d1925e0..df042a056b13 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -227,20 +227,32 @@ namespace ccf LedgerSecretWrappingKey combine_from_submitted_shares(Store::Tx& tx) { - auto submitted_shares = tx.get_view(network.submitted_shares)->get(0); - if (!submitted_shares.has_value()) - { - throw std::logic_error("Failed to retrieve current submitted shares"); - } + auto [submitted_shares_view, config_view] = + tx.get_view(network.submitted_shares, network.config); std::vector shares; - for (auto const& s : submitted_shares.value()) + submitted_shares_view->foreach( + [&shares, this]( + const MemberId member_id, + const std::vector& encrypted_share) { + SecretSharing::Share share; + auto decrypted_share = decrypt_submitted_share(encrypted_share); + std::copy_n( + decrypted_share.begin(), + SecretSharing::SHARE_LENGTH, + share.begin()); + shares.emplace_back(share); + return true; + }); + + auto recovery_threshold = config_view->get(0)->recovery_threshold; + if (recovery_threshold > shares.size()) { - SecretSharing::Share share; - auto decrypted_share = decrypt_submitted_share(s.second); - std::copy_n( - decrypted_share.begin(), SecretSharing::SHARE_LENGTH, share.begin()); - shares.emplace_back(share); + throw std::logic_error(fmt::format( + "Error combining recovery shares: only {} recovery shares were " + "submitted but recovery threshold is {}", + shares.size(), + recovery_threshold)); } return LedgerSecretWrappingKey( @@ -379,36 +391,34 @@ namespace ccf throw std::logic_error("Failed to get active service"); } - auto submitted_shares = submitted_shares_view->get(0); - if (!submitted_shares.has_value()) - { - throw std::logic_error( - "Failed to get current submitted recovery shares"); - } - - auto submitted_shares_map = submitted_shares.value(); - submitted_shares_map[member_id] = - encrypt_submitted_share(submitted_recovery_share); - - submitted_shares_view->put(0, submitted_shares_map); - - LOG_FAIL_FMT( - "submitted_shares_map.size() {}", submitted_shares_map.size()); - - return submitted_shares_map.size(); + auto submitted_shares = submitted_shares_view->put( + member_id, encrypt_submitted_share(submitted_recovery_share)); + + size_t submitted_shares_count = 0; + submitted_shares_view->foreach( + [&submitted_shares_count]( + const MemberId member_id, + const std::vector& encrypted_share) { + LOG_FAIL_FMT("Submitted share for member {}", member_id); + submitted_shares_count++; + return true; + }); + + LOG_FAIL_FMT("submitted_shares_count: {}", submitted_shares_count); + return submitted_shares_count; } void clear_submitted_recovery_shares(Store::Tx& tx) { auto submitted_shares_view = tx.get_view(network.submitted_shares); - auto submitted_shares = submitted_shares_view->get(0); - if (!submitted_shares.has_value()) - { - throw std::logic_error( - "Failed to get current submitted recovery shares"); - } - submitted_shares_view->put(0, {}); + submitted_shares_view->foreach( + [&submitted_shares_view]( + const MemberId member_id, + const std::vector& encrypted_share) { + submitted_shares_view->remove(member_id); + return true; + }); } }; } \ No newline at end of file diff --git a/src/node/submitted_shares.h b/src/node/submitted_shares.h index 634d487c22a8..b54441a3d878 100644 --- a/src/node/submitted_shares.h +++ b/src/node/submitted_shares.h @@ -8,10 +8,11 @@ namespace ccf { - // The key for this table will always be 0 as there can only be one recovery - // happening at any given time - // TODO: Use the member ID for key instead? Probably more efficient? + // This table keeps track of the submitted encrypted recovery share so that + // the public-only service is resilient to elections while members submit + // their recovery shares. + // Because shares are submitted to the public-only network on recovery, this + // table is public but the shares are encrypted with the latest ledger secret. - using SubmittedShares = - Store::Map>>; + using SubmittedShares = Store::Map>; } \ No newline at end of file From 8560a2ccf77a587221351a82adc3cc8e5bf87a68 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Thu, 21 May 2020 18:14:27 +0100 Subject: [PATCH 12/33] Refactoring for better unit testing --- src/enclave/enclave.h | 7 +- src/node/node_state.h | 10 +- src/node/rpc/member_frontend.h | 14 +- src/node/rpc/node_interface.h | 4 - src/node/rpc/test/member_voting_test.cpp | 3021 +++++++++++----------- src/node/rpc/test/node_stub.h | 62 +- 6 files changed, 1550 insertions(+), 1568 deletions(-) diff --git a/src/enclave/enclave.h b/src/enclave/enclave.h index dfb7e7d9c7f2..d42e7e2bd60c 100644 --- a/src/enclave/enclave.h +++ b/src/enclave/enclave.h @@ -27,6 +27,7 @@ namespace enclave ringbuffer::WriterFactory basic_writer_factory; oversized::WriterFactory writer_factory; ccf::NetworkState network; + ccf::ShareManager share_manager; std::shared_ptr n2n_channels; ccf::Notifier notifier; ccf::Timers timers; @@ -53,7 +54,9 @@ namespace enclave notifier(writer_factory), rpc_map(std::make_shared()), rpcsessions(std::make_shared(writer_factory, rpc_map)), - node(writer_factory, network, rpcsessions, notifier, timers), + share_manager(network), + node( + writer_factory, network, rpcsessions, notifier, timers, share_manager), cmd_forwarder(std::make_shared>( rpcsessions, n2n_channels, rpc_map)), consensus_type(consensus_type_) @@ -64,7 +67,7 @@ namespace enclave REGISTER_FRONTEND( rpc_map, members, - std::make_unique(network, node)); + std::make_unique(network, node, share_manager)); REGISTER_FRONTEND( rpc_map, users, ccfapp::get_rpc_handler(network, notifier)); diff --git a/src/node/node_state.h b/src/node/node_state.h index 0c15e82261e5..559695c28895 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -209,7 +209,8 @@ namespace ccf NetworkState& network, std::shared_ptr rpcsessions, ccf::Notifier& notifier, - Timers& timers) : + Timers& timers, + ShareManager& share_manager) : sm(State::uninitialized), self(INVALID_ID), node_sign_kp(tls::make_key_pair()), @@ -220,7 +221,7 @@ namespace ccf rpcsessions(rpcsessions), notifier(notifier), timers(timers), - share_manager(network) + share_manager(share_manager) { ::EverCrypt_AutoConfig2_init(); } @@ -1026,11 +1027,6 @@ namespace ccf }); }; - ShareManager& get_share_manager() override - { - return share_manager; - } - void restore_ledger_secrets(Store::Tx& tx) override { try diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index c7741a49e7dd..58636640c97e 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -546,11 +546,14 @@ namespace ccf const lua::TxScriptRunner tsr; public: - MemberHandlers(NetworkTables& network, AbstractNodeState& node) : + MemberHandlers( + NetworkTables& network, + AbstractNodeState& node, + ShareManager& share_manager) : CommonHandlerRegistry(*network.tables, Tables::MEMBER_CERTS), network(network), node(node), - share_manager(node.get_share_manager()), + share_manager(share_manager), tsr(network) {} @@ -1021,10 +1024,13 @@ namespace ccf Members* members; public: - MemberRpcFrontend(NetworkTables& network, AbstractNodeState& node) : + MemberRpcFrontend( + NetworkTables& network, + AbstractNodeState& node, + ShareManager& share_manager) : RpcFrontend( *network.tables, member_handlers, &network.member_client_signatures), - member_handlers(network, node), + member_handlers(network, node, share_manager), members(&network.members) {} diff --git a/src/node/rpc/node_interface.h b/src/node/rpc/node_interface.h index c3d3a6c2a501..ff1c16b73df5 100644 --- a/src/node/rpc/node_interface.h +++ b/src/node/rpc/node_interface.h @@ -27,10 +27,6 @@ namespace ccf virtual NodeId get_node_id() const = 0; virtual kv::Version get_last_recovered_commit_idx() = 0; - // TODO: Can call share_manager directly - // virtual bool split_ledger_secrets(Store::Tx& tx) = 0; - - virtual ShareManager& get_share_manager() = 0; virtual void restore_ledger_secrets(Store::Tx& tx) = 0; }; diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 200de9521150..85f46c014d1c 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -64,11 +64,13 @@ T parse_response_body(const TResponse& r) try { body_j = jsonrpc::unpack(r.body, jsonrpc::Pack::Text); + LOG_FAIL_FMT("RPC resp: {}", body_j.dump()); } catch (const nlohmann::json::parse_error& e) { std::cerr << e.what() << std::endl; - std::cerr << std::string(r.body.begin(), r.body.end()) << std::endl; + std::cerr << "RPC error: " << std::string(r.body.begin(), r.body.end()) + << std::endl; } return body_j.get(); @@ -180,1408 +182,1427 @@ std::vector get_cert_data(uint64_t member_id, tls::KeyPairPtr& kp_mem) return kp_mem->self_sign("CN=new member" + to_string(member_id)); } -auto init_frontend( - NetworkTables& network, - GenesisGenerator& gen, - StubNodeState& node, - const int n_members, - std::vector>& member_certs) -{ - // create members - for (uint8_t i = 0; i < n_members; i++) - { - member_certs.push_back(get_cert_data(i, kp)); - gen.add_member(member_certs.back(), {}); - } - - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); - gen.finalize(); - - return MemberRpcFrontend(network, node); -} - -DOCTEST_TEST_CASE("Member query/read") -{ - // initialize the network state - NetworkTables network; - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node; - MemberRpcFrontend frontend(network, node); - frontend.open(); - const auto member_id = gen.add_member(member_cert, {}); - gen.finalize(); - - const enclave::SessionContext member_session( - enclave::InvalidSessionId, member_cert); - - // put value to read - constexpr auto key = 123; - constexpr auto value = 456; - Store::Tx tx; - tx.get_view(network.values)->put(key, value); - DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - - static constexpr auto query = R"xxx( - local tables = ... - return tables["ccf.values"]:get(123) - )xxx"; - - DOCTEST_SUBCASE("Query: bytecode/script allowed access") - { - // set member ACL so that the VALUES table is accessible - Store::Tx tx; - tx.get_view(network.whitelists) - ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); - DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - - bool compile = true; - do - { - const auto req = create_request(query_params(query, compile), "query"); - const auto r = frontend_process(frontend, req, member_cert); - const auto result = parse_response_body(r); - DOCTEST_CHECK(result == value); - compile = !compile; - } while (!compile); - } - - DOCTEST_SUBCASE("Query: table not in ACL") - { - // set member ACL so that no table is accessible - Store::Tx tx; - tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); - DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - - auto req = create_request(query_params(query, true), "query"); - const auto response = frontend_process(frontend, req, member_cert); - - check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); - } - - DOCTEST_SUBCASE("Read: allowed access, key exists") - { - Store::Tx tx; - tx.get_view(network.whitelists) - ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); - DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - - auto read_call = - create_request(read_params(key, Tables::VALUES), "read"); - const auto r = frontend_process(frontend, read_call, member_cert); - const auto result = parse_response_body(r); - DOCTEST_CHECK(result == value); - } - - DOCTEST_SUBCASE("Read: allowed access, key doesn't exist") - { - constexpr auto wrong_key = 321; - Store::Tx tx; - tx.get_view(network.whitelists) - ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); - DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - - auto read_call = - create_request(read_params(wrong_key, Tables::VALUES), "read"); - const auto response = frontend_process(frontend, read_call, member_cert); - - check_error(response, HTTP_STATUS_BAD_REQUEST); - } - - DOCTEST_SUBCASE("Read: access not allowed") - { - Store::Tx tx; - tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); - DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - - auto read_call = - create_request(read_params(key, Tables::VALUES), "read"); - const auto response = frontend_process(frontend, read_call, member_cert); - - check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); - } -} - -DOCTEST_TEST_CASE("Proposer ballot") -{ - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - - const auto proposer_cert = get_cert_data(0, kp); - const auto proposer_id = gen.add_member(proposer_cert, {}); - const auto voter_cert = get_cert_data(1, kp); - const auto voter_id = gen.add_member(voter_cert, {}); - - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); - gen.finalize(); - - StubNodeState node; - MemberRpcFrontend frontend(network, node); - frontend.open(); - - size_t proposal_id; - - const ccf::Script vote_for("return true"); - const ccf::Script vote_against("return false"); - { - DOCTEST_INFO("Propose, initially voting against"); - - const auto proposed_member = get_cert_data(2, kp); - - Propose::In proposal; - proposal.script = std::string(R"xxx( - tables, member_info = ... - return Calls:call("new_member", member_info) - )xxx"); - proposal.parameter["cert"] = proposed_member; - proposal.parameter["keyshare"] = dummy_key_share; - proposal.ballot = vote_against; - const auto propose = create_signed_request(proposal, "propose", kp); - const auto r = frontend_process(frontend, propose, proposer_cert); - - // the proposal should be accepted, but not succeed immediately - const auto result = parse_response_body(r); - DOCTEST_CHECK(result.state == ProposalState::OPEN); - - proposal_id = result.proposal_id; - } - - { - DOCTEST_INFO("Second member votes for proposal"); - - const auto vote = - create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); - const auto r = frontend_process(frontend, vote, voter_cert); - - // The vote should not yet succeed - check_result_state(r, ProposalState::OPEN); - } - - { - DOCTEST_INFO("Read current votes"); - - const auto proposal_result = - get_proposal(frontend, proposal_id, proposer_cert); - - const auto& votes = proposal_result.votes; - DOCTEST_CHECK(votes.size() == 2); - - const auto proposer_vote = votes.find(proposer_id); - DOCTEST_CHECK(proposer_vote != votes.end()); - DOCTEST_CHECK(proposer_vote->second == vote_against); - - const auto voter_vote = votes.find(voter_id); - DOCTEST_CHECK(voter_vote != votes.end()); - DOCTEST_CHECK(voter_vote->second == vote_for); - } - - { - DOCTEST_INFO("Proposer votes for"); - - const auto vote = - create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); - const auto r = frontend_process(frontend, vote, proposer_cert); - - // The vote should now succeed - check_result_state(r, ProposalState::ACCEPTED); - } -} - -struct NewMember -{ - MemberId id; - tls::KeyPairPtr kp = tls::make_key_pair(); - Cert cert; -}; - -DOCTEST_TEST_CASE("Add new members until there are 7 then reject") -{ - logger::config::level() = logger::INFO; - - constexpr auto initial_members = 3; - constexpr auto n_new_members = 7; - constexpr auto max_members = 8; - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node(std::make_shared(network)); - // add three initial active members - // the proposer - auto proposer_id = gen.add_member(member_cert, {}); - - // the voters - const auto voter_a_cert = get_cert_data(1, kp); - auto voter_a = gen.add_member(voter_a_cert, {}); - const auto voter_b_cert = get_cert_data(2, kp); - auto voter_b = gen.add_member(voter_b_cert, {}); - - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); - gen.set_recovery_threshold(1); - gen.open_service(); - gen.finalize(); - MemberRpcFrontend frontend(network, node); - frontend.open(); - - vector new_members(n_new_members); - - auto i = 0ul; - for (auto& new_member : new_members) - { - const auto proposal_id = i; - new_member.id = initial_members + i++; - - // new member certificate - auto cert_pem = - new_member.kp->self_sign(fmt::format("CN=new member{}", new_member.id)); - auto keyshare = dummy_key_share; - auto v = tls::make_verifier(cert_pem); - const auto _cert = v->raw(); - new_member.cert = {_cert->raw.p, _cert->raw.p + _cert->raw.len}; - - // check new_member id does not work before member is added - const auto read_next_req = create_request( - read_params(ValueIds::NEXT_MEMBER_ID, Tables::VALUES), "read"); - const auto r = frontend_process(frontend, read_next_req, new_member.cert); - check_error(r, HTTP_STATUS_FORBIDDEN); - - // propose new member, as proposer - Propose::In proposal; - proposal.script = std::string(R"xxx( - tables, member_info = ... - return Calls:call("new_member", member_info) - )xxx"); - proposal.parameter["cert"] = cert_pem; - proposal.parameter["keyshare"] = keyshare; - - const auto propose = create_signed_request(proposal, "propose", kp); - - { - const auto r = frontend_process(frontend, propose, member_cert); - const auto result = parse_response_body(r); - - // the proposal should be accepted, but not succeed immediately - DOCTEST_CHECK(result.proposal_id == proposal_id); - DOCTEST_CHECK(result.state == ProposalState::OPEN); - } - - // read initial proposal, as second member - const Proposal initial_read = - get_proposal(frontend, proposal_id, voter_a_cert); - DOCTEST_CHECK(initial_read.proposer == proposer_id); - DOCTEST_CHECK(initial_read.script == proposal.script); - DOCTEST_CHECK(initial_read.parameter == proposal.parameter); - - // vote as second member - Script vote_ballot(fmt::format( - R"xxx( - local tables, calls = ... - local n = 0 - tables["ccf.members"]:foreach( function(k, v) n = n + 1 end ) - if n < {} then - return true - else - return false - end - )xxx", - max_members)); - - const auto vote = - create_signed_request(Vote{proposal_id, vote_ballot}, "vote", kp); - - { - const auto r = frontend_process(frontend, vote, voter_a_cert); - const auto result = parse_response_body(r); - - if (new_member.id < max_members) - { - // vote should succeed - DOCTEST_CHECK(result.state == ProposalState::ACCEPTED); - // check that member with the new new_member cert can make RPCs now - DOCTEST_CHECK( - parse_response_body(frontend_process( - frontend, read_next_req, new_member.cert)) == new_member.id + 1); - - // successful proposals are removed from the kv, so we can't confirm - // their final state - } - else - { - // vote should not succeed - DOCTEST_CHECK(result.state == ProposalState::OPEN); - // check that member with the new new_member cert can make RPCs now - check_error( - frontend_process(frontend, read_next_req, new_member.cert), - HTTP_STATUS_FORBIDDEN); - - // re-read proposal, as second member - const Proposal final_read = - get_proposal(frontend, proposal_id, voter_a_cert); - DOCTEST_CHECK(final_read.proposer == proposer_id); - DOCTEST_CHECK(final_read.script == proposal.script); - DOCTEST_CHECK(final_read.parameter == proposal.parameter); - - const auto my_vote = final_read.votes.find(voter_a); - DOCTEST_CHECK(my_vote != final_read.votes.end()); - DOCTEST_CHECK(my_vote->second == vote_ballot); - } - } - } - - DOCTEST_SUBCASE("ACK from newly added members") - { - // iterate over all new_members, except for the last one - for (auto new_member = new_members.cbegin(); new_member != - new_members.cend() - (initial_members + n_new_members - max_members); - new_member++) - { - // (1) read ack entry - const auto read_state_digest_req = create_request( - read_params(new_member->id, Tables::MEMBER_ACKS), "read"); - const auto ack0 = parse_response_body( - frontend_process(frontend, read_state_digest_req, new_member->cert)); - DOCTEST_REQUIRE(std::all_of( - ack0.state_digest.begin(), ack0.state_digest.end(), [](uint8_t i) { - return i == 0; - })); - - { - // make sure that there is a signature in the signatures table since - // ack's depend on that - Store::Tx tx; - auto sig_view = tx.get_view(network.signatures); - Signature sig_value; - sig_view->put(0, sig_value); - DOCTEST_REQUIRE(tx.commit() == kv::CommitSuccess::OK); - } - - // (2) ask for a fresher digest of state - const auto freshen_state_digest_req = - create_request(nullptr, "updateAckStateDigest"); - const auto freshen_state_digest = parse_response_body( - frontend_process(frontend, freshen_state_digest_req, new_member->cert)); - DOCTEST_CHECK(freshen_state_digest.state_digest != ack0.state_digest); - - // (3) read ack entry again and check that the state digest has changed - const auto ack1 = parse_response_body( - frontend_process(frontend, read_state_digest_req, new_member->cert)); - DOCTEST_CHECK(ack0.state_digest != ack1.state_digest); - DOCTEST_CHECK(freshen_state_digest.state_digest == ack1.state_digest); - - // (4) sign stale state and send it - StateDigest params; - params.state_digest = ack0.state_digest; - const auto send_stale_sig_req = - create_signed_request(params, "ack", new_member->kp); - check_error( - frontend_process(frontend, send_stale_sig_req, new_member->cert), - HTTP_STATUS_BAD_REQUEST); - - // (5) sign new state digest and send it - params.state_digest = ack1.state_digest; - const auto send_good_sig_req = - create_signed_request(params, "ack", new_member->kp); - const auto good_response = - frontend_process(frontend, send_good_sig_req, new_member->cert); - DOCTEST_CHECK(good_response.status == HTTP_STATUS_OK); - DOCTEST_CHECK(parse_response_body(good_response)); - - // (6) read own member status - const auto read_status_req = - create_request(read_params(new_member->id, Tables::MEMBERS), "read"); - const auto mi = parse_response_body( - frontend_process(frontend, read_status_req, new_member->cert)); - DOCTEST_CHECK(mi.status == MemberStatus::ACTIVE); - } - } -} - -DOCTEST_TEST_CASE("Accept node") -{ - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node; - auto new_kp = tls::make_key_pair(); - - const auto member_0_cert = get_cert_data(0, new_kp); - const auto member_1_cert = get_cert_data(1, kp); - const auto member_0 = gen.add_member(member_0_cert, {}); - const auto member_1 = gen.add_member(member_1_cert, {}); - - // node to be tested - // new node certificate - auto new_ca = new_kp->self_sign("CN=new node"); - NodeInfo ni; - ni.cert = new_ca; - gen.add_node(ni); - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); - gen.finalize(); - MemberRpcFrontend frontend(network, node); - frontend.open(); - auto node_id = 0; - - // check node exists with status pending - { - auto read_values = - create_request(read_params(node_id, Tables::NODES), "read"); - const auto r = parse_response_body( - frontend_process(frontend, read_values, member_0_cert)); - - DOCTEST_CHECK(r.status == NodeStatus::PENDING); - } - - // m0 proposes adding new node - { - Script proposal(R"xxx( - local tables, node_id = ... - return Calls:call("trust_node", node_id) - )xxx"); - const auto propose = - create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, member_0_cert)); - - DOCTEST_CHECK(r.state == ProposalState::OPEN); - DOCTEST_CHECK(r.proposal_id == 0); - } - - // m1 votes for accepting a single new node - { - Script vote_ballot(R"xxx( - local tables, calls = ... - return #calls == 1 and calls[1].func == "trust_node" - )xxx"); - const auto vote = create_signed_request(Vote{0, vote_ballot}, "vote", kp); - - check_result_state( - frontend_process(frontend, vote, member_1_cert), ProposalState::ACCEPTED); - } - - // check node exists with status pending - { - const auto read_values = - create_request(read_params(node_id, Tables::NODES), "read"); - const auto r = parse_response_body( - frontend_process(frontend, read_values, member_0_cert)); - DOCTEST_CHECK(r.status == NodeStatus::TRUSTED); - } - - // m0 proposes retire node - { - Script proposal(R"xxx( - local tables, node_id = ... - return Calls:call("retire_node", node_id) - )xxx"); - const auto propose = - create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, member_0_cert)); - - DOCTEST_CHECK(r.state == ProposalState::OPEN); - DOCTEST_CHECK(r.proposal_id == 1); - } - - // m1 votes for retiring node - { - const Script vote_ballot("return true"); - const auto vote = create_signed_request(Vote{1, vote_ballot}, "vote", kp); - check_result_state( - frontend_process(frontend, vote, member_1_cert), ProposalState::ACCEPTED); - } - - // check that node exists with status retired - { - auto read_values = - create_request(read_params(node_id, Tables::NODES), "read"); - const auto r = parse_response_body( - frontend_process(frontend, read_values, member_0_cert)); - DOCTEST_CHECK(r.status == NodeStatus::RETIRED); - } - - // check that retired node cannot be trusted - { - Script proposal(R"xxx( - local tables, node_id = ... - return Calls:call("trust_node", node_id) - )xxx"); - const auto propose = - create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, member_0_cert)); - - const Script vote_ballot("return true"); - const auto vote = - create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); - check_result_state( - frontend_process(frontend, vote, member_1_cert), ProposalState::FAILED); - } - - // check that retired node cannot be retired again - { - Script proposal(R"xxx( - local tables, node_id = ... - return Calls:call("retire_node", node_id) - )xxx"); - const auto propose = - create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, member_0_cert)); - - const Script vote_ballot("return true"); - const auto vote = - create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); - check_result_state( - frontend_process(frontend, vote, member_1_cert), ProposalState::FAILED); - } -} - -ProposalInfo test_raw_writes( - NetworkTables& network, - GenesisGenerator& gen, - StubNodeState& node, - Propose::In proposal, - const int n_members = 1, - const int pro_votes = 1, - bool explicit_proposer_vote = false) -{ - std::vector> member_certs; - auto frontend = init_frontend(network, gen, node, n_members, member_certs); - frontend.open(); - - // check values before - { - Store::Tx tx; - auto next_member_id_r = - tx.get_view(network.values)->get(ValueIds::NEXT_MEMBER_ID); - DOCTEST_CHECK(next_member_id_r); - DOCTEST_CHECK(*next_member_id_r == n_members); - } - - // propose - const auto proposal_id = 0ul; - { - const uint8_t proposer_id = 0; - const auto propose = create_signed_request(proposal, "propose", kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, member_certs[0])); - - const auto expected_state = - (n_members == 1) ? ProposalState::ACCEPTED : ProposalState::OPEN; - DOCTEST_CHECK(r.state == expected_state); - DOCTEST_CHECK(r.proposal_id == proposal_id); - if (r.state == ProposalState::ACCEPTED) - return r; - } - - // con votes - for (int i = n_members - 1; i >= pro_votes; i--) - { - const Script vote("return false"); - const auto vote_serialized = - create_signed_request(Vote{proposal_id, vote}, "vote", kp); - - check_result_state( - frontend_process(frontend, vote_serialized, member_certs[i]), - ProposalState::OPEN); - } - - // pro votes (proposer also votes) - ProposalInfo info = {}; - for (uint8_t i = explicit_proposer_vote ? 0 : 1; i < pro_votes; i++) - { - const Script vote("return true"); - const auto vote_serialized = - create_signed_request(Vote{proposal_id, vote}, "vote", kp); - if (info.state == ProposalState::OPEN) - { - info = parse_response_body( - frontend_process(frontend, vote_serialized, member_certs[i])); - } - else - { - // proposal has been accepted - additional votes return an error - check_error( - frontend_process(frontend, vote_serialized, member_certs[i]), - HTTP_STATUS_BAD_REQUEST); - } - } - return info; -} - -DOCTEST_TEST_CASE("Propose raw writes") -{ - logger::config::level() = logger::INFO; - DOCTEST_SUBCASE("insensitive tables") - { - const auto n_members = 3; - for (int pro_votes = 0; pro_votes <= n_members; pro_votes++) - { - const bool should_succeed = pro_votes > n_members / 2; - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node; - nlohmann::json recovery_threshold = 4; - - Store::Tx tx_before; - auto configuration = tx_before.get_view(network.config)->get(0); - DOCTEST_REQUIRE_FALSE(configuration.has_value()); - - const auto expected_state = - should_succeed ? ProposalState::ACCEPTED : ProposalState::OPEN; - const auto proposal_info = test_raw_writes( - network, - gen, - node, - {R"xxx( - local tables, recovery_threshold = ... - local p = Puts:new() - p:put("ccf.config", 0, {recovery_threshold = recovery_threshold}) - return Calls:call("raw_puts", p) - )xxx"s, - 4}, - n_members, - pro_votes); - DOCTEST_CHECK(proposal_info.state == expected_state); - if (!should_succeed) - continue; - - // check results - Store::Tx tx_after; - configuration = tx_after.get_view(network.config)->get(0); - DOCTEST_CHECK(configuration.has_value()); - DOCTEST_CHECK(configuration->recovery_threshold == recovery_threshold); - } - } - - DOCTEST_SUBCASE("sensitive tables") - { - // propose changes to sensitive tables; changes must only be accepted - // unanimously create new network for each case - const auto sensitive_tables = {Tables::WHITELISTS, Tables::GOV_SCRIPTS}; - const auto n_members = 3; - // let proposer vote/not vote - for (const auto proposer_vote : {true, false}) - { - for (int pro_votes = 0; pro_votes < n_members; pro_votes++) - { - for (const auto& sensitive_table : sensitive_tables) - { - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node; - - const auto sensitive_put = - "return Calls:call('raw_puts', Puts:put('"s + sensitive_table + - "', 9, {'aaa'}))"s; - const auto expected_state = (n_members == pro_votes) ? - ProposalState::ACCEPTED : - ProposalState::OPEN; - const auto proposal_info = test_raw_writes( - network, - gen, - node, - {sensitive_put}, - n_members, - pro_votes, - proposer_vote); - DOCTEST_CHECK(proposal_info.state == expected_state); - } - } - } - } -} - -DOCTEST_TEST_CASE("Remove proposal") -{ - NewMember caller; - auto cert = caller.kp->self_sign("CN=new member"); - auto v = tls::make_verifier(cert); - caller.cert = v->der_cert_data(); - - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - - StubNodeState node; - gen.add_member(member_cert, {}); - gen.add_member(cert, {}); - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); - gen.finalize(); - MemberRpcFrontend frontend(network, node); - frontend.open(); - auto proposal_id = 0; - auto wrong_proposal_id = 1; - ccf::Script proposal_script(R"xxx( - local tables, param = ... - return {} - )xxx"); - - // check that the proposal doesn't exist - { - Store::Tx tx; - auto proposal = tx.get_view(network.proposals)->get(proposal_id); - DOCTEST_CHECK(!proposal); - } - - { - const auto propose = - create_signed_request(Propose::In{proposal_script, 0}, "propose", kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, member_cert)); - - DOCTEST_CHECK(r.proposal_id == proposal_id); - DOCTEST_CHECK(r.state == ProposalState::OPEN); - } - - // check that the proposal is there - { - Store::Tx tx; - auto proposal = tx.get_view(network.proposals)->get(proposal_id); - DOCTEST_CHECK(proposal); - DOCTEST_CHECK(proposal->state == ProposalState::OPEN); - DOCTEST_CHECK( - proposal->script.text.value() == proposal_script.text.value()); - } - - DOCTEST_SUBCASE("Attempt withdraw proposal with non existing id") - { - json param; - param["id"] = wrong_proposal_id; - const auto withdraw = create_signed_request(param, "withdraw", kp); - - check_error( - frontend_process(frontend, withdraw, member_cert), - HTTP_STATUS_BAD_REQUEST); - } - - DOCTEST_SUBCASE("Attempt withdraw proposal that you didn't propose") - { - json param; - param["id"] = proposal_id; - const auto withdraw = create_signed_request(param, "withdraw", caller.kp); - - check_error( - frontend_process(frontend, withdraw, cert), HTTP_STATUS_FORBIDDEN); - } - - DOCTEST_SUBCASE("Successfully withdraw proposal") - { - json param; - param["id"] = proposal_id; - const auto withdraw = create_signed_request(param, "withdraw", kp); - - check_result_state( - frontend_process(frontend, withdraw, member_cert), - ProposalState::WITHDRAWN); - - // check that the proposal is now withdrawn - { - Store::Tx tx; - auto proposal = tx.get_view(network.proposals)->get(proposal_id); - DOCTEST_CHECK(proposal.has_value()); - DOCTEST_CHECK(proposal->state == ProposalState::WITHDRAWN); - } - } -} - -DOCTEST_TEST_CASE("Complete proposal after initial rejection") -{ - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node; - std::vector> member_certs; - auto frontend = init_frontend(network, gen, node, 3, member_certs); - frontend.open(); - - { - DOCTEST_INFO("Propose"); - const auto proposal = - "return Calls:call('raw_puts', Puts:put('ccf.values', 999, 999))"s; - const auto propose = - create_signed_request(Propose::In{proposal}, "propose", kp); - - Store::Tx tx; - const auto r = parse_response_body( - frontend_process(frontend, propose, member_certs[0])); - DOCTEST_CHECK(r.state == ProposalState::OPEN); - } - - { - DOCTEST_INFO("Vote that rejects initially"); - const Script vote(R"xxx( - local tables = ... - return tables["ccf.values"]:get(123) == 123 - )xxx"); - const auto vote_serialized = - create_signed_request(Vote{0, vote}, "vote", kp); - - check_result_state( - frontend_process(frontend, vote_serialized, member_certs[1]), - ProposalState::OPEN); - } - - { - DOCTEST_INFO("Try to complete"); - const auto complete = - create_signed_request(ProposalAction{0}, "complete", kp); - - check_result_state( - frontend_process(frontend, complete, member_certs[1]), - ProposalState::OPEN); - } - - { - DOCTEST_INFO("Put value that makes vote agree"); - Store::Tx tx; - tx.get_view(network.values)->put(123, 123); - DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - } - - { - DOCTEST_INFO("Try again to complete"); - const auto complete = - create_signed_request(ProposalAction{0}, "complete", kp); - - check_result_state( - frontend_process(frontend, complete, member_certs[1]), - ProposalState::ACCEPTED); - } -} - -DOCTEST_TEST_CASE("Vetoed proposal gets rejected") -{ - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node; - const auto voter_a_cert = get_cert_data(1, kp); - auto voter_a = gen.add_member(voter_a_cert, {}); - const auto voter_b_cert = get_cert_data(2, kp); - auto voter_b = gen.add_member(voter_b_cert, {}); - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_veto_script_file)); - gen.finalize(); - MemberRpcFrontend frontend(network, node); - frontend.open(); - - Script proposal(R"xxx( - tables, user_cert = ... - return Calls:call("new_user", user_cert) - )xxx"); - - const vector user_cert = kp->self_sign("CN=new user"); - const auto propose = - create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); - - const auto r = parse_response_body( - frontend_process(frontend, propose, voter_a_cert)); - DOCTEST_CHECK(r.state == ProposalState::OPEN); - DOCTEST_CHECK(r.proposal_id == 0); - - const ccf::Script vote_against("return false"); - { - DOCTEST_INFO("Member vetoes proposal"); - - const auto vote = create_signed_request(Vote{0, vote_against}, "vote", kp); - const auto r = frontend_process(frontend, vote, voter_b_cert); - - check_result_state(r, ProposalState::REJECTED); - } - - { - DOCTEST_INFO("Check proposal was rejected"); - - const auto proposal = get_proposal(frontend, 0, voter_a_cert); - - DOCTEST_CHECK(proposal.state == ProposalState::REJECTED); - } -} - -DOCTEST_TEST_CASE("Add user via proposed call") -{ - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - StubNodeState node; - const auto member_cert = get_cert_data(0, kp); - gen.add_member(member_cert, {}); - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); - gen.finalize(); - MemberRpcFrontend frontend(network, node); - frontend.open(); - - Script proposal(R"xxx( - tables, user_cert = ... - return Calls:call("new_user", user_cert) - )xxx"); - - const vector user_cert = kp->self_sign("CN=new user"); - const auto propose = - create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); - - const auto r = parse_response_body( - frontend_process(frontend, propose, member_cert)); - DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); - DOCTEST_CHECK(r.proposal_id == 0); - - Store::Tx tx1; - const auto uid = tx1.get_view(network.values)->get(ValueIds::NEXT_USER_ID); - DOCTEST_CHECK(uid); - DOCTEST_CHECK(*uid == 1); - const auto uid1 = tx1.get_view(network.user_certs) - ->get(tls::make_verifier(user_cert)->der_cert_data()); - DOCTEST_CHECK(uid1); - DOCTEST_CHECK(*uid1 == 0); -} - -DOCTEST_TEST_CASE("Passing members ballot with operator") -{ - // Members pass a ballot with a constitution that includes an operator - // Operator votes, but is _not_ taken into consideration - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - - // Operating member, as set in operator_gov.lua - const auto operator_cert = get_cert_data(0, kp); - const auto operator_id = gen.add_member(operator_cert, {}); - - // Non-operating members - std::map members; - for (size_t i = 1; i < 4; i++) - { - auto cert = get_cert_data(i, kp); - members[gen.add_member(cert, {})] = cert; - } - - set_whitelists(gen); - gen.set_gov_scripts( - lua::Interpreter().invoke(operator_gov_script_file)); - gen.finalize(); - - StubNodeState node; - MemberRpcFrontend frontend(network, node); - frontend.open(); - - size_t proposal_id; - size_t proposer_id = 1; - size_t voter_id = 2; - - const ccf::Script vote_for("return true"); - const ccf::Script vote_against("return false"); - { - DOCTEST_INFO("Propose and vote for"); - - const auto proposed_member = get_cert_data(4, kp); - - Propose::In proposal; - proposal.script = std::string(R"xxx( - tables, member_info = ... - return Calls:call("new_member", member_info) - )xxx"); - proposal.parameter["cert"] = proposed_member; - proposal.parameter["keyshare"] = dummy_key_share; - proposal.ballot = vote_for; - - const auto propose = create_signed_request(proposal, "propose", kp); - const auto r = parse_response_body(frontend_process( - frontend, - propose, - tls::make_verifier(members[proposer_id])->der_cert_data())); - - DOCTEST_CHECK(r.state == ProposalState::OPEN); - - proposal_id = r.proposal_id; - } - - { - DOCTEST_INFO("Operator votes, but without effect"); - - const auto vote = - create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); - const auto r = frontend_process(frontend, vote, operator_cert); - - check_result_state(r, ProposalState::OPEN); - } - - { - DOCTEST_INFO("Second member votes for proposal, which passes"); - - const auto vote = - create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); - const auto r = frontend_process(frontend, vote, members[voter_id]); - - check_result_state(r, ProposalState::ACCEPTED); - } - - { - DOCTEST_INFO("Validate vote tally"); - - const auto readj = create_signed_request( - read_params(proposal_id, Tables::PROPOSALS), "read", kp); - - const auto proposal = - get_proposal(frontend, proposal_id, members[proposer_id]); - - const auto& votes = proposal.votes; - DOCTEST_CHECK(votes.size() == 3); - - const auto operator_vote = votes.find(operator_id); - DOCTEST_CHECK(operator_vote != votes.end()); - DOCTEST_CHECK(operator_vote->second == vote_for); - - const auto proposer_vote = votes.find(proposer_id); - DOCTEST_CHECK(proposer_vote != votes.end()); - DOCTEST_CHECK(proposer_vote->second == vote_for); - - const auto voter_vote = votes.find(voter_id); - DOCTEST_CHECK(voter_vote != votes.end()); - DOCTEST_CHECK(voter_vote->second == vote_for); - } -} - -DOCTEST_TEST_CASE("Passing operator vote") -{ - // Operator issues a proposal that only requires its own vote - // and gets it through without member votes - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - auto new_kp = tls::make_key_pair(); - auto new_ca = new_kp->self_sign("CN=new node"); - NodeInfo ni; - ni.cert = new_ca; - gen.add_node(ni); - - // Operating member, as set in operator_gov.lua - const auto operator_cert = get_cert_data(0, kp); - const auto operator_id = gen.add_member(operator_cert, {}); - - // Non-operating members - std::map members; - for (size_t i = 1; i < 4; i++) - { - auto cert = get_cert_data(i, kp); - members[gen.add_member(cert, {})] = cert; - } - - set_whitelists(gen); - gen.set_gov_scripts( - lua::Interpreter().invoke(operator_gov_script_file)); - gen.finalize(); - - StubNodeState node; - MemberRpcFrontend frontend(network, node); - frontend.open(); - - size_t proposal_id; - - const ccf::Script vote_for("return true"); - const ccf::Script vote_against("return false"); - - auto node_id = 0; - { - DOCTEST_INFO("Check node exists with status pending"); - auto read_values = - create_request(read_params(node_id, Tables::NODES), "read"); - const auto r = parse_response_body( - frontend_process(frontend, read_values, operator_cert)); - - DOCTEST_CHECK(r.status == NodeStatus::PENDING); - } - - { - DOCTEST_INFO("Operator proposes and votes for node"); - Script proposal(R"xxx( - local tables, node_id = ... - return Calls:call("trust_node", node_id) - )xxx"); - - const auto propose = create_signed_request( - Propose::In{proposal, node_id, vote_for}, "propose", kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, operator_cert)); - - DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); - proposal_id = r.proposal_id; - } - - { - DOCTEST_INFO("Validate vote tally"); - - const auto readj = create_signed_request( - read_params(proposal_id, Tables::PROPOSALS), "read", kp); - - const auto proposal = get_proposal(frontend, proposal_id, operator_cert); - - const auto& votes = proposal.votes; - DOCTEST_CHECK(votes.size() == 1); - - const auto proposer_vote = votes.find(operator_id); - DOCTEST_CHECK(proposer_vote != votes.end()); - DOCTEST_CHECK(proposer_vote->second == vote_for); - } -} - -DOCTEST_TEST_CASE("Members passing an operator vote") -{ - // Operator proposes a vote, but does not vote for it - // A majority of members pass the vote - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - auto new_kp = tls::make_key_pair(); - auto new_ca = new_kp->self_sign("CN=new node"); - NodeInfo ni; - ni.cert = new_ca; - gen.add_node(ni); - - // Operating member, as set in operator_gov.lua - const auto operator_cert = get_cert_data(0, kp); - const auto operator_id = gen.add_member(operator_cert, {}); - - // Non-operating members - std::map members; - for (size_t i = 1; i < 4; i++) - { - auto cert = get_cert_data(i, kp); - members[gen.add_member(cert, {})] = cert; - } - - set_whitelists(gen); - gen.set_gov_scripts( - lua::Interpreter().invoke(operator_gov_script_file)); - gen.finalize(); - - StubNodeState node; - MemberRpcFrontend frontend(network, node); - frontend.open(); - - size_t proposal_id; - - const ccf::Script vote_for("return true"); - const ccf::Script vote_against("return false"); - - auto node_id = 0; - { - DOCTEST_INFO("Check node exists with status pending"); - const auto read_values = - create_request(read_params(node_id, Tables::NODES), "read"); - const auto r = parse_response_body( - frontend_process(frontend, read_values, operator_cert)); - DOCTEST_CHECK(r.status == NodeStatus::PENDING); - } - - { - DOCTEST_INFO("Operator proposes and votes against adding node"); - Script proposal(R"xxx( - local tables, node_id = ... - return Calls:call("trust_node", node_id) - )xxx"); - - const auto propose = create_signed_request( - Propose::In{proposal, node_id, vote_against}, "propose", kp); - const auto r = parse_response_body( - frontend_process(frontend, propose, operator_cert)); - - DOCTEST_CHECK(r.state == ProposalState::OPEN); - proposal_id = r.proposal_id; - } - - size_t first_voter_id = 1; - size_t second_voter_id = 2; - - { - DOCTEST_INFO("First member votes for proposal"); - - const auto vote = - create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); - const auto r = frontend_process(frontend, vote, members[first_voter_id]); - - check_result_state(r, ProposalState::OPEN); - } - - { - DOCTEST_INFO("Second member votes for proposal"); - - const auto vote = - create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); - const auto r = frontend_process(frontend, vote, members[second_voter_id]); - - check_result_state(r, ProposalState::ACCEPTED); - } - - { - DOCTEST_INFO("Validate vote tally"); - - const auto readj = create_signed_request( - read_params(proposal_id, Tables::PROPOSALS), "read", kp); - - const auto proposal = get_proposal(frontend, proposal_id, operator_cert); - - const auto& votes = proposal.votes; - DOCTEST_CHECK(votes.size() == 3); - - const auto proposer_vote = votes.find(operator_id); - DOCTEST_CHECK(proposer_vote != votes.end()); - DOCTEST_CHECK(proposer_vote->second == vote_against); - - const auto first_vote = votes.find(first_voter_id); - DOCTEST_CHECK(first_vote != votes.end()); - DOCTEST_CHECK(first_vote->second == vote_for); - - const auto second_vote = votes.find(second_voter_id); - DOCTEST_CHECK(second_vote != votes.end()); - DOCTEST_CHECK(second_vote->second == vote_for); - } -} - -DOCTEST_TEST_CASE("User data") -{ - NetworkTables network; - network.tables->set_encryptor(encryptor); - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - const auto member_id = gen.add_member(member_cert, {}); - const auto user_id = gen.add_user(user_cert); - set_whitelists(gen); - gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); - gen.finalize(); - - StubNodeState node; - MemberRpcFrontend frontend(network, node); - frontend.open(); - - const auto read_user_info = - create_request(read_params(user_id, Tables::USERS), "read"); - - { - DOCTEST_INFO("user data is initially empty"); - const auto read_response = parse_response_body( - frontend_process(frontend, read_user_info, member_cert)); - DOCTEST_CHECK(read_response.user_data.is_null()); - } - - { - auto user_data_object = nlohmann::json::object(); - user_data_object["name"] = "bob"; - user_data_object["permissions"] = {"read", "delete"}; - - DOCTEST_INFO("user data can be set to an object"); - Propose::In proposal; - proposal.script = fmt::format( - R"xxx( - proposed_user_data = {{ - name = "bob", - permissions = {{"read", "delete"}} - }} - return Calls:call("set_user_data", {{user_id = {}, user_data = - proposed_user_data}}) - )xxx", - user_id); - const auto proposal_serialized = - create_signed_request(proposal, "propose", kp); - const auto propose_response = parse_response_body( - frontend_process(frontend, proposal_serialized, member_cert)); - DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); - - DOCTEST_INFO("user data object can be read"); - const auto read_response = parse_response_body( - frontend_process(frontend, read_user_info, member_cert)); - DOCTEST_CHECK(read_response.user_data == user_data_object); - } - - { - const auto user_data_string = "ADMINISTRATOR"; - - DOCTEST_INFO("user data can be overwritten"); - Propose::In proposal; - proposal.script = std::string(R"xxx( - local tables, param = ... - return Calls:call("set_user_data", {user_id = param.id, user_data = - param.data}) - )xxx"); - proposal.parameter["id"] = user_id; - proposal.parameter["data"] = user_data_string; - const auto proposal_serialized = - create_signed_request(proposal, "propose", kp); - const auto propose_response = parse_response_body( - frontend_process(frontend, proposal_serialized, member_cert)); - DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); - - DOCTEST_INFO("user data object can be read"); - const auto response = parse_response_body( - frontend_process(frontend, read_user_info, member_cert)); - DOCTEST_CHECK(response.user_data == user_data_string); - } -} +// auto init_frontend( +// NetworkTables& network, +// GenesisGenerator& gen, +// StubNodeState& node, +// const int n_members, +// std::vector>& member_certs) +// { +// // create members +// for (uint8_t i = 0; i < n_members; i++) +// { +// member_certs.push_back(get_cert_data(i, kp)); +// gen.add_member(member_certs.back(), {}); +// } + +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); +// gen.finalize(); + +// return MemberRpcFrontend(network, node); +// } + +// DOCTEST_TEST_CASE("Member query/read") +// { +// // initialize the network state +// NetworkTables network; +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node; +// MemberRpcFrontend frontend(network, node); +// frontend.open(); +// const auto member_id = gen.add_member(member_cert, {}); +// gen.finalize(); + +// const enclave::SessionContext member_session( +// enclave::InvalidSessionId, member_cert); + +// // put value to read +// constexpr auto key = 123; +// constexpr auto value = 456; +// Store::Tx tx; +// tx.get_view(network.values)->put(key, value); +// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + +// static constexpr auto query = R"xxx( +// local tables = ... +// return tables["ccf.values"]:get(123) +// )xxx"; + +// DOCTEST_SUBCASE("Query: bytecode/script allowed access") +// { +// // set member ACL so that the VALUES table is accessible +// Store::Tx tx; +// tx.get_view(network.whitelists) +// ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); +// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + +// bool compile = true; +// do +// { +// const auto req = create_request(query_params(query, compile), "query"); +// const auto r = frontend_process(frontend, req, member_cert); +// const auto result = parse_response_body(r); +// DOCTEST_CHECK(result == value); +// compile = !compile; +// } while (!compile); +// } + +// DOCTEST_SUBCASE("Query: table not in ACL") +// { +// // set member ACL so that no table is accessible +// Store::Tx tx; +// tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); +// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + +// auto req = create_request(query_params(query, true), "query"); +// const auto response = frontend_process(frontend, req, member_cert); + +// check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); +// } + +// DOCTEST_SUBCASE("Read: allowed access, key exists") +// { +// Store::Tx tx; +// tx.get_view(network.whitelists) +// ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); +// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + +// auto read_call = +// create_request(read_params(key, Tables::VALUES), "read"); +// const auto r = frontend_process(frontend, read_call, member_cert); +// const auto result = parse_response_body(r); +// DOCTEST_CHECK(result == value); +// } + +// DOCTEST_SUBCASE("Read: allowed access, key doesn't exist") +// { +// constexpr auto wrong_key = 321; +// Store::Tx tx; +// tx.get_view(network.whitelists) +// ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); +// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + +// auto read_call = +// create_request(read_params(wrong_key, Tables::VALUES), "read"); +// const auto response = frontend_process(frontend, read_call, member_cert); + +// check_error(response, HTTP_STATUS_BAD_REQUEST); +// } + +// DOCTEST_SUBCASE("Read: access not allowed") +// { +// Store::Tx tx; +// tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); +// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + +// auto read_call = +// create_request(read_params(key, Tables::VALUES), "read"); +// const auto response = frontend_process(frontend, read_call, member_cert); + +// check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); +// } +// } + +// DOCTEST_TEST_CASE("Proposer ballot") +// { +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); + +// const auto proposer_cert = get_cert_data(0, kp); +// const auto proposer_id = gen.add_member(proposer_cert, {}); +// const auto voter_cert = get_cert_data(1, kp); +// const auto voter_id = gen.add_member(voter_cert, {}); + +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); +// gen.finalize(); + +// StubNodeState node; +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// size_t proposal_id; + +// const ccf::Script vote_for("return true"); +// const ccf::Script vote_against("return false"); +// { +// DOCTEST_INFO("Propose, initially voting against"); + +// const auto proposed_member = get_cert_data(2, kp); + +// Propose::In proposal; +// proposal.script = std::string(R"xxx( +// tables, member_info = ... +// return Calls:call("new_member", member_info) +// )xxx"); +// proposal.parameter["cert"] = proposed_member; +// proposal.parameter["keyshare"] = dummy_key_share; +// proposal.ballot = vote_against; +// const auto propose = create_signed_request(proposal, "propose", kp); +// const auto r = frontend_process(frontend, propose, proposer_cert); + +// // the proposal should be accepted, but not succeed immediately +// const auto result = parse_response_body(r); +// DOCTEST_CHECK(result.state == ProposalState::OPEN); + +// proposal_id = result.proposal_id; +// } + +// { +// DOCTEST_INFO("Second member votes for proposal"); + +// const auto vote = +// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); +// const auto r = frontend_process(frontend, vote, voter_cert); + +// // The vote should not yet succeed +// check_result_state(r, ProposalState::OPEN); +// } + +// { +// DOCTEST_INFO("Read current votes"); + +// const auto proposal_result = +// get_proposal(frontend, proposal_id, proposer_cert); + +// const auto& votes = proposal_result.votes; +// DOCTEST_CHECK(votes.size() == 2); + +// const auto proposer_vote = votes.find(proposer_id); +// DOCTEST_CHECK(proposer_vote != votes.end()); +// DOCTEST_CHECK(proposer_vote->second == vote_against); + +// const auto voter_vote = votes.find(voter_id); +// DOCTEST_CHECK(voter_vote != votes.end()); +// DOCTEST_CHECK(voter_vote->second == vote_for); +// } + +// { +// DOCTEST_INFO("Proposer votes for"); + +// const auto vote = +// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); +// const auto r = frontend_process(frontend, vote, proposer_cert); + +// // The vote should now succeed +// check_result_state(r, ProposalState::ACCEPTED); +// } +// } + +// struct NewMember +// { +// MemberId id; +// tls::KeyPairPtr kp = tls::make_key_pair(); +// Cert cert; +// }; + +// DOCTEST_TEST_CASE("Add new members until there are 7 then reject") +// { +// logger::config::level() = logger::INFO; + +// constexpr auto initial_members = 3; +// constexpr auto n_new_members = 7; +// constexpr auto max_members = 8; +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node(std::make_shared(network)); +// // add three initial active members +// // the proposer +// auto proposer_id = gen.add_member(member_cert, {}); + +// // the voters +// const auto voter_a_cert = get_cert_data(1, kp); +// auto voter_a = gen.add_member(voter_a_cert, {}); +// const auto voter_b_cert = get_cert_data(2, kp); +// auto voter_b = gen.add_member(voter_b_cert, {}); + +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); +// gen.set_recovery_threshold(1); +// gen.open_service(); +// gen.finalize(); +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// vector new_members(n_new_members); + +// auto i = 0ul; +// for (auto& new_member : new_members) +// { +// const auto proposal_id = i; +// new_member.id = initial_members + i++; + +// // new member certificate +// auto cert_pem = +// new_member.kp->self_sign(fmt::format("CN=new member{}", +// new_member.id)); +// auto keyshare = dummy_key_share; +// auto v = tls::make_verifier(cert_pem); +// const auto _cert = v->raw(); +// new_member.cert = {_cert->raw.p, _cert->raw.p + _cert->raw.len}; + +// // check new_member id does not work before member is added +// const auto read_next_req = create_request( +// read_params(ValueIds::NEXT_MEMBER_ID, Tables::VALUES), "read"); +// const auto r = frontend_process(frontend, read_next_req, +// new_member.cert); check_error(r, HTTP_STATUS_FORBIDDEN); + +// // propose new member, as proposer +// Propose::In proposal; +// proposal.script = std::string(R"xxx( +// tables, member_info = ... +// return Calls:call("new_member", member_info) +// )xxx"); +// proposal.parameter["cert"] = cert_pem; +// proposal.parameter["keyshare"] = keyshare; + +// const auto propose = create_signed_request(proposal, "propose", kp); + +// { +// const auto r = frontend_process(frontend, propose, member_cert); +// const auto result = parse_response_body(r); + +// // the proposal should be accepted, but not succeed immediately +// DOCTEST_CHECK(result.proposal_id == proposal_id); +// DOCTEST_CHECK(result.state == ProposalState::OPEN); +// } + +// // read initial proposal, as second member +// const Proposal initial_read = +// get_proposal(frontend, proposal_id, voter_a_cert); +// DOCTEST_CHECK(initial_read.proposer == proposer_id); +// DOCTEST_CHECK(initial_read.script == proposal.script); +// DOCTEST_CHECK(initial_read.parameter == proposal.parameter); + +// // vote as second member +// Script vote_ballot(fmt::format( +// R"xxx( +// local tables, calls = ... +// local n = 0 +// tables["ccf.members"]:foreach( function(k, v) n = n + 1 end ) +// if n < {} then +// return true +// else +// return false +// end +// )xxx", +// max_members)); + +// const auto vote = +// create_signed_request(Vote{proposal_id, vote_ballot}, "vote", kp); + +// { +// const auto r = frontend_process(frontend, vote, voter_a_cert); +// const auto result = parse_response_body(r); + +// if (new_member.id < max_members) +// { +// // vote should succeed +// DOCTEST_CHECK(result.state == ProposalState::ACCEPTED); +// // check that member with the new new_member cert can make RPCs now +// DOCTEST_CHECK( +// parse_response_body(frontend_process( +// frontend, read_next_req, new_member.cert)) == new_member.id + 1); + +// // successful proposals are removed from the kv, so we can't confirm +// // their final state +// } +// else +// { +// // vote should not succeed +// DOCTEST_CHECK(result.state == ProposalState::OPEN); +// // check that member with the new new_member cert can make RPCs now +// check_error( +// frontend_process(frontend, read_next_req, new_member.cert), +// HTTP_STATUS_FORBIDDEN); + +// // re-read proposal, as second member +// const Proposal final_read = +// get_proposal(frontend, proposal_id, voter_a_cert); +// DOCTEST_CHECK(final_read.proposer == proposer_id); +// DOCTEST_CHECK(final_read.script == proposal.script); +// DOCTEST_CHECK(final_read.parameter == proposal.parameter); + +// const auto my_vote = final_read.votes.find(voter_a); +// DOCTEST_CHECK(my_vote != final_read.votes.end()); +// DOCTEST_CHECK(my_vote->second == vote_ballot); +// } +// } +// } + +// DOCTEST_SUBCASE("ACK from newly added members") +// { +// // iterate over all new_members, except for the last one +// for (auto new_member = new_members.cbegin(); new_member != +// new_members.cend() - (initial_members + n_new_members - +// max_members); new_member++) +// { +// // (1) read ack entry +// const auto read_state_digest_req = create_request( +// read_params(new_member->id, Tables::MEMBER_ACKS), "read"); +// const auto ack0 = parse_response_body( +// frontend_process(frontend, read_state_digest_req, new_member->cert)); +// DOCTEST_REQUIRE(std::all_of( +// ack0.state_digest.begin(), ack0.state_digest.end(), [](uint8_t i) { +// return i == 0; +// })); + +// { +// // make sure that there is a signature in the signatures table since +// // ack's depend on that +// Store::Tx tx; +// auto sig_view = tx.get_view(network.signatures); +// Signature sig_value; +// sig_view->put(0, sig_value); +// DOCTEST_REQUIRE(tx.commit() == kv::CommitSuccess::OK); +// } + +// // (2) ask for a fresher digest of state +// const auto freshen_state_digest_req = +// create_request(nullptr, "updateAckStateDigest"); +// const auto freshen_state_digest = parse_response_body( +// frontend_process(frontend, freshen_state_digest_req, +// new_member->cert)); +// DOCTEST_CHECK(freshen_state_digest.state_digest != ack0.state_digest); + +// // (3) read ack entry again and check that the state digest has changed +// const auto ack1 = parse_response_body( +// frontend_process(frontend, read_state_digest_req, new_member->cert)); +// DOCTEST_CHECK(ack0.state_digest != ack1.state_digest); +// DOCTEST_CHECK(freshen_state_digest.state_digest == ack1.state_digest); + +// // (4) sign stale state and send it +// StateDigest params; +// params.state_digest = ack0.state_digest; +// const auto send_stale_sig_req = +// create_signed_request(params, "ack", new_member->kp); +// check_error( +// frontend_process(frontend, send_stale_sig_req, new_member->cert), +// HTTP_STATUS_BAD_REQUEST); + +// // (5) sign new state digest and send it +// params.state_digest = ack1.state_digest; +// const auto send_good_sig_req = +// create_signed_request(params, "ack", new_member->kp); +// const auto good_response = +// frontend_process(frontend, send_good_sig_req, new_member->cert); +// DOCTEST_CHECK(good_response.status == HTTP_STATUS_OK); +// DOCTEST_CHECK(parse_response_body(good_response)); + +// // (6) read own member status +// const auto read_status_req = +// create_request(read_params(new_member->id, Tables::MEMBERS), "read"); +// const auto mi = parse_response_body( +// frontend_process(frontend, read_status_req, new_member->cert)); +// DOCTEST_CHECK(mi.status == MemberStatus::ACTIVE); +// } +// } +// } + +// DOCTEST_TEST_CASE("Accept node") +// { +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node; +// auto new_kp = tls::make_key_pair(); + +// const auto member_0_cert = get_cert_data(0, new_kp); +// const auto member_1_cert = get_cert_data(1, kp); +// const auto member_0 = gen.add_member(member_0_cert, {}); +// const auto member_1 = gen.add_member(member_1_cert, {}); + +// // node to be tested +// // new node certificate +// auto new_ca = new_kp->self_sign("CN=new node"); +// NodeInfo ni; +// ni.cert = new_ca; +// gen.add_node(ni); +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); +// gen.finalize(); +// MemberRpcFrontend frontend(network, node); +// frontend.open(); +// auto node_id = 0; + +// // check node exists with status pending +// { +// auto read_values = +// create_request(read_params(node_id, Tables::NODES), "read"); +// const auto r = parse_response_body( +// frontend_process(frontend, read_values, member_0_cert)); + +// DOCTEST_CHECK(r.status == NodeStatus::PENDING); +// } + +// // m0 proposes adding new node +// { +// Script proposal(R"xxx( +// local tables, node_id = ... +// return Calls:call("trust_node", node_id) +// )xxx"); +// const auto propose = +// create_signed_request(Propose::In{proposal, node_id}, "propose", +// new_kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_0_cert)); + +// DOCTEST_CHECK(r.state == ProposalState::OPEN); +// DOCTEST_CHECK(r.proposal_id == 0); +// } + +// // m1 votes for accepting a single new node +// { +// Script vote_ballot(R"xxx( +// local tables, calls = ... +// return #calls == 1 and calls[1].func == "trust_node" +// )xxx"); +// const auto vote = create_signed_request(Vote{0, vote_ballot}, "vote", +// kp); + +// check_result_state( +// frontend_process(frontend, vote, member_1_cert), +// ProposalState::ACCEPTED); +// } + +// // check node exists with status pending +// { +// const auto read_values = +// create_request(read_params(node_id, Tables::NODES), "read"); +// const auto r = parse_response_body( +// frontend_process(frontend, read_values, member_0_cert)); +// DOCTEST_CHECK(r.status == NodeStatus::TRUSTED); +// } + +// // m0 proposes retire node +// { +// Script proposal(R"xxx( +// local tables, node_id = ... +// return Calls:call("retire_node", node_id) +// )xxx"); +// const auto propose = +// create_signed_request(Propose::In{proposal, node_id}, "propose", +// new_kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_0_cert)); + +// DOCTEST_CHECK(r.state == ProposalState::OPEN); +// DOCTEST_CHECK(r.proposal_id == 1); +// } + +// // m1 votes for retiring node +// { +// const Script vote_ballot("return true"); +// const auto vote = create_signed_request(Vote{1, vote_ballot}, "vote", +// kp); check_result_state( +// frontend_process(frontend, vote, member_1_cert), +// ProposalState::ACCEPTED); +// } + +// // check that node exists with status retired +// { +// auto read_values = +// create_request(read_params(node_id, Tables::NODES), "read"); +// const auto r = parse_response_body( +// frontend_process(frontend, read_values, member_0_cert)); +// DOCTEST_CHECK(r.status == NodeStatus::RETIRED); +// } + +// // check that retired node cannot be trusted +// { +// Script proposal(R"xxx( +// local tables, node_id = ... +// return Calls:call("trust_node", node_id) +// )xxx"); +// const auto propose = +// create_signed_request(Propose::In{proposal, node_id}, "propose", +// new_kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_0_cert)); + +// const Script vote_ballot("return true"); +// const auto vote = +// create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); +// check_result_state( +// frontend_process(frontend, vote, member_1_cert), +// ProposalState::FAILED); +// } + +// // check that retired node cannot be retired again +// { +// Script proposal(R"xxx( +// local tables, node_id = ... +// return Calls:call("retire_node", node_id) +// )xxx"); +// const auto propose = +// create_signed_request(Propose::In{proposal, node_id}, "propose", +// new_kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_0_cert)); + +// const Script vote_ballot("return true"); +// const auto vote = +// create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); +// check_result_state( +// frontend_process(frontend, vote, member_1_cert), +// ProposalState::FAILED); +// } +// } + +// ProposalInfo test_raw_writes( +// NetworkTables& network, +// GenesisGenerator& gen, +// StubNodeState& node, +// Propose::In proposal, +// const int n_members = 1, +// const int pro_votes = 1, +// bool explicit_proposer_vote = false) +// { +// std::vector> member_certs; +// auto frontend = init_frontend(network, gen, node, n_members, member_certs); +// frontend.open(); + +// // check values before +// { +// Store::Tx tx; +// auto next_member_id_r = +// tx.get_view(network.values)->get(ValueIds::NEXT_MEMBER_ID); +// DOCTEST_CHECK(next_member_id_r); +// DOCTEST_CHECK(*next_member_id_r == n_members); +// } + +// // propose +// const auto proposal_id = 0ul; +// { +// const uint8_t proposer_id = 0; +// const auto propose = create_signed_request(proposal, "propose", kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_certs[0])); + +// const auto expected_state = +// (n_members == 1) ? ProposalState::ACCEPTED : ProposalState::OPEN; +// DOCTEST_CHECK(r.state == expected_state); +// DOCTEST_CHECK(r.proposal_id == proposal_id); +// if (r.state == ProposalState::ACCEPTED) +// return r; +// } + +// // con votes +// for (int i = n_members - 1; i >= pro_votes; i--) +// { +// const Script vote("return false"); +// const auto vote_serialized = +// create_signed_request(Vote{proposal_id, vote}, "vote", kp); + +// check_result_state( +// frontend_process(frontend, vote_serialized, member_certs[i]), +// ProposalState::OPEN); +// } + +// // pro votes (proposer also votes) +// ProposalInfo info = {}; +// for (uint8_t i = explicit_proposer_vote ? 0 : 1; i < pro_votes; i++) +// { +// const Script vote("return true"); +// const auto vote_serialized = +// create_signed_request(Vote{proposal_id, vote}, "vote", kp); +// if (info.state == ProposalState::OPEN) +// { +// info = parse_response_body( +// frontend_process(frontend, vote_serialized, member_certs[i])); +// } +// else +// { +// // proposal has been accepted - additional votes return an error +// check_error( +// frontend_process(frontend, vote_serialized, member_certs[i]), +// HTTP_STATUS_BAD_REQUEST); +// } +// } +// return info; +// } + +// DOCTEST_TEST_CASE("Propose raw writes") +// { +// logger::config::level() = logger::INFO; +// DOCTEST_SUBCASE("insensitive tables") +// { +// const auto n_members = 3; +// for (int pro_votes = 0; pro_votes <= n_members; pro_votes++) +// { +// const bool should_succeed = pro_votes > n_members / 2; +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node; +// nlohmann::json recovery_threshold = 4; + +// Store::Tx tx_before; +// auto configuration = tx_before.get_view(network.config)->get(0); +// DOCTEST_REQUIRE_FALSE(configuration.has_value()); + +// const auto expected_state = +// should_succeed ? ProposalState::ACCEPTED : ProposalState::OPEN; +// const auto proposal_info = test_raw_writes( +// network, +// gen, +// node, +// {R"xxx( +// local tables, recovery_threshold = ... +// local p = Puts:new() +// p:put("ccf.config", 0, {recovery_threshold = recovery_threshold}) +// return Calls:call("raw_puts", p) +// )xxx"s, +// 4}, +// n_members, +// pro_votes); +// DOCTEST_CHECK(proposal_info.state == expected_state); +// if (!should_succeed) +// continue; + +// // check results +// Store::Tx tx_after; +// configuration = tx_after.get_view(network.config)->get(0); +// DOCTEST_CHECK(configuration.has_value()); +// DOCTEST_CHECK(configuration->recovery_threshold == recovery_threshold); +// } +// } + +// DOCTEST_SUBCASE("sensitive tables") +// { +// // propose changes to sensitive tables; changes must only be accepted +// // unanimously create new network for each case +// const auto sensitive_tables = {Tables::WHITELISTS, Tables::GOV_SCRIPTS}; +// const auto n_members = 3; +// // let proposer vote/not vote +// for (const auto proposer_vote : {true, false}) +// { +// for (int pro_votes = 0; pro_votes < n_members; pro_votes++) +// { +// for (const auto& sensitive_table : sensitive_tables) +// { +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node; + +// const auto sensitive_put = +// "return Calls:call('raw_puts', Puts:put('"s + sensitive_table + +// "', 9, {'aaa'}))"s; +// const auto expected_state = (n_members == pro_votes) ? +// ProposalState::ACCEPTED : +// ProposalState::OPEN; +// const auto proposal_info = test_raw_writes( +// network, +// gen, +// node, +// {sensitive_put}, +// n_members, +// pro_votes, +// proposer_vote); +// DOCTEST_CHECK(proposal_info.state == expected_state); +// } +// } +// } +// } +// } + +// DOCTEST_TEST_CASE("Remove proposal") +// { +// NewMember caller; +// auto cert = caller.kp->self_sign("CN=new member"); +// auto v = tls::make_verifier(cert); +// caller.cert = v->der_cert_data(); + +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); + +// StubNodeState node; +// gen.add_member(member_cert, {}); +// gen.add_member(cert, {}); +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); +// gen.finalize(); +// MemberRpcFrontend frontend(network, node); +// frontend.open(); +// auto proposal_id = 0; +// auto wrong_proposal_id = 1; +// ccf::Script proposal_script(R"xxx( +// local tables, param = ... +// return {} +// )xxx"); + +// // check that the proposal doesn't exist +// { +// Store::Tx tx; +// auto proposal = tx.get_view(network.proposals)->get(proposal_id); +// DOCTEST_CHECK(!proposal); +// } + +// { +// const auto propose = +// create_signed_request(Propose::In{proposal_script, 0}, "propose", kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_cert)); + +// DOCTEST_CHECK(r.proposal_id == proposal_id); +// DOCTEST_CHECK(r.state == ProposalState::OPEN); +// } + +// // check that the proposal is there +// { +// Store::Tx tx; +// auto proposal = tx.get_view(network.proposals)->get(proposal_id); +// DOCTEST_CHECK(proposal); +// DOCTEST_CHECK(proposal->state == ProposalState::OPEN); +// DOCTEST_CHECK( +// proposal->script.text.value() == proposal_script.text.value()); +// } + +// DOCTEST_SUBCASE("Attempt withdraw proposal with non existing id") +// { +// json param; +// param["id"] = wrong_proposal_id; +// const auto withdraw = create_signed_request(param, "withdraw", kp); + +// check_error( +// frontend_process(frontend, withdraw, member_cert), +// HTTP_STATUS_BAD_REQUEST); +// } + +// DOCTEST_SUBCASE("Attempt withdraw proposal that you didn't propose") +// { +// json param; +// param["id"] = proposal_id; +// const auto withdraw = create_signed_request(param, "withdraw", +// caller.kp); + +// check_error( +// frontend_process(frontend, withdraw, cert), HTTP_STATUS_FORBIDDEN); +// } + +// DOCTEST_SUBCASE("Successfully withdraw proposal") +// { +// json param; +// param["id"] = proposal_id; +// const auto withdraw = create_signed_request(param, "withdraw", kp); + +// check_result_state( +// frontend_process(frontend, withdraw, member_cert), +// ProposalState::WITHDRAWN); + +// // check that the proposal is now withdrawn +// { +// Store::Tx tx; +// auto proposal = tx.get_view(network.proposals)->get(proposal_id); +// DOCTEST_CHECK(proposal.has_value()); +// DOCTEST_CHECK(proposal->state == ProposalState::WITHDRAWN); +// } +// } +// } + +// DOCTEST_TEST_CASE("Complete proposal after initial rejection") +// { +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node; +// std::vector> member_certs; +// auto frontend = init_frontend(network, gen, node, 3, member_certs); +// frontend.open(); + +// { +// DOCTEST_INFO("Propose"); +// const auto proposal = +// "return Calls:call('raw_puts', Puts:put('ccf.values', 999, 999))"s; +// const auto propose = +// create_signed_request(Propose::In{proposal}, "propose", kp); + +// Store::Tx tx; +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_certs[0])); +// DOCTEST_CHECK(r.state == ProposalState::OPEN); +// } + +// { +// DOCTEST_INFO("Vote that rejects initially"); +// const Script vote(R"xxx( +// local tables = ... +// return tables["ccf.values"]:get(123) == 123 +// )xxx"); +// const auto vote_serialized = +// create_signed_request(Vote{0, vote}, "vote", kp); + +// check_result_state( +// frontend_process(frontend, vote_serialized, member_certs[1]), +// ProposalState::OPEN); +// } + +// { +// DOCTEST_INFO("Try to complete"); +// const auto complete = +// create_signed_request(ProposalAction{0}, "complete", kp); + +// check_result_state( +// frontend_process(frontend, complete, member_certs[1]), +// ProposalState::OPEN); +// } + +// { +// DOCTEST_INFO("Put value that makes vote agree"); +// Store::Tx tx; +// tx.get_view(network.values)->put(123, 123); +// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); +// } + +// { +// DOCTEST_INFO("Try again to complete"); +// const auto complete = +// create_signed_request(ProposalAction{0}, "complete", kp); + +// check_result_state( +// frontend_process(frontend, complete, member_certs[1]), +// ProposalState::ACCEPTED); +// } +// } + +// DOCTEST_TEST_CASE("Vetoed proposal gets rejected") +// { +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node; +// const auto voter_a_cert = get_cert_data(1, kp); +// auto voter_a = gen.add_member(voter_a_cert, {}); +// const auto voter_b_cert = get_cert_data(2, kp); +// auto voter_b = gen.add_member(voter_b_cert, {}); +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_veto_script_file)); +// gen.finalize(); +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// Script proposal(R"xxx( +// tables, user_cert = ... +// return Calls:call("new_user", user_cert) +// )xxx"); + +// const vector user_cert = kp->self_sign("CN=new user"); +// const auto propose = +// create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); + +// const auto r = parse_response_body( +// frontend_process(frontend, propose, voter_a_cert)); +// DOCTEST_CHECK(r.state == ProposalState::OPEN); +// DOCTEST_CHECK(r.proposal_id == 0); + +// const ccf::Script vote_against("return false"); +// { +// DOCTEST_INFO("Member vetoes proposal"); + +// const auto vote = create_signed_request(Vote{0, vote_against}, "vote", +// kp); const auto r = frontend_process(frontend, vote, voter_b_cert); + +// check_result_state(r, ProposalState::REJECTED); +// } + +// { +// DOCTEST_INFO("Check proposal was rejected"); + +// const auto proposal = get_proposal(frontend, 0, voter_a_cert); + +// DOCTEST_CHECK(proposal.state == ProposalState::REJECTED); +// } +// } + +// DOCTEST_TEST_CASE("Add user via proposed call") +// { +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// StubNodeState node; +// const auto member_cert = get_cert_data(0, kp); +// gen.add_member(member_cert, {}); +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); +// gen.finalize(); +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// Script proposal(R"xxx( +// tables, user_cert = ... +// return Calls:call("new_user", user_cert) +// )xxx"); + +// const vector user_cert = kp->self_sign("CN=new user"); +// const auto propose = +// create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); + +// const auto r = parse_response_body( +// frontend_process(frontend, propose, member_cert)); +// DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); +// DOCTEST_CHECK(r.proposal_id == 0); + +// Store::Tx tx1; +// const auto uid = tx1.get_view(network.values)->get(ValueIds::NEXT_USER_ID); +// DOCTEST_CHECK(uid); +// DOCTEST_CHECK(*uid == 1); +// const auto uid1 = tx1.get_view(network.user_certs) +// ->get(tls::make_verifier(user_cert)->der_cert_data()); +// DOCTEST_CHECK(uid1); +// DOCTEST_CHECK(*uid1 == 0); +// } + +// DOCTEST_TEST_CASE("Passing members ballot with operator") +// { +// // Members pass a ballot with a constitution that includes an operator +// // Operator votes, but is _not_ taken into consideration +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); + +// // Operating member, as set in operator_gov.lua +// const auto operator_cert = get_cert_data(0, kp); +// const auto operator_id = gen.add_member(operator_cert, {}); + +// // Non-operating members +// std::map members; +// for (size_t i = 1; i < 4; i++) +// { +// auto cert = get_cert_data(i, kp); +// members[gen.add_member(cert, {})] = cert; +// } + +// set_whitelists(gen); +// gen.set_gov_scripts( +// lua::Interpreter().invoke(operator_gov_script_file)); +// gen.finalize(); + +// StubNodeState node; +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// size_t proposal_id; +// size_t proposer_id = 1; +// size_t voter_id = 2; + +// const ccf::Script vote_for("return true"); +// const ccf::Script vote_against("return false"); +// { +// DOCTEST_INFO("Propose and vote for"); + +// const auto proposed_member = get_cert_data(4, kp); + +// Propose::In proposal; +// proposal.script = std::string(R"xxx( +// tables, member_info = ... +// return Calls:call("new_member", member_info) +// )xxx"); +// proposal.parameter["cert"] = proposed_member; +// proposal.parameter["keyshare"] = dummy_key_share; +// proposal.ballot = vote_for; + +// const auto propose = create_signed_request(proposal, "propose", kp); +// const auto r = parse_response_body(frontend_process( +// frontend, +// propose, +// tls::make_verifier(members[proposer_id])->der_cert_data())); + +// DOCTEST_CHECK(r.state == ProposalState::OPEN); + +// proposal_id = r.proposal_id; +// } + +// { +// DOCTEST_INFO("Operator votes, but without effect"); + +// const auto vote = +// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); +// const auto r = frontend_process(frontend, vote, operator_cert); + +// check_result_state(r, ProposalState::OPEN); +// } + +// { +// DOCTEST_INFO("Second member votes for proposal, which passes"); + +// const auto vote = +// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); +// const auto r = frontend_process(frontend, vote, members[voter_id]); + +// check_result_state(r, ProposalState::ACCEPTED); +// } + +// { +// DOCTEST_INFO("Validate vote tally"); + +// const auto readj = create_signed_request( +// read_params(proposal_id, Tables::PROPOSALS), "read", kp); + +// const auto proposal = +// get_proposal(frontend, proposal_id, members[proposer_id]); + +// const auto& votes = proposal.votes; +// DOCTEST_CHECK(votes.size() == 3); + +// const auto operator_vote = votes.find(operator_id); +// DOCTEST_CHECK(operator_vote != votes.end()); +// DOCTEST_CHECK(operator_vote->second == vote_for); + +// const auto proposer_vote = votes.find(proposer_id); +// DOCTEST_CHECK(proposer_vote != votes.end()); +// DOCTEST_CHECK(proposer_vote->second == vote_for); + +// const auto voter_vote = votes.find(voter_id); +// DOCTEST_CHECK(voter_vote != votes.end()); +// DOCTEST_CHECK(voter_vote->second == vote_for); +// } +// } + +// DOCTEST_TEST_CASE("Passing operator vote") +// { +// // Operator issues a proposal that only requires its own vote +// // and gets it through without member votes +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// auto new_kp = tls::make_key_pair(); +// auto new_ca = new_kp->self_sign("CN=new node"); +// NodeInfo ni; +// ni.cert = new_ca; +// gen.add_node(ni); + +// // Operating member, as set in operator_gov.lua +// const auto operator_cert = get_cert_data(0, kp); +// const auto operator_id = gen.add_member(operator_cert, {}); + +// // Non-operating members +// std::map members; +// for (size_t i = 1; i < 4; i++) +// { +// auto cert = get_cert_data(i, kp); +// members[gen.add_member(cert, {})] = cert; +// } + +// set_whitelists(gen); +// gen.set_gov_scripts( +// lua::Interpreter().invoke(operator_gov_script_file)); +// gen.finalize(); + +// StubNodeState node; +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// size_t proposal_id; + +// const ccf::Script vote_for("return true"); +// const ccf::Script vote_against("return false"); + +// auto node_id = 0; +// { +// DOCTEST_INFO("Check node exists with status pending"); +// auto read_values = +// create_request(read_params(node_id, Tables::NODES), "read"); +// const auto r = parse_response_body( +// frontend_process(frontend, read_values, operator_cert)); + +// DOCTEST_CHECK(r.status == NodeStatus::PENDING); +// } + +// { +// DOCTEST_INFO("Operator proposes and votes for node"); +// Script proposal(R"xxx( +// local tables, node_id = ... +// return Calls:call("trust_node", node_id) +// )xxx"); + +// const auto propose = create_signed_request( +// Propose::In{proposal, node_id, vote_for}, "propose", kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, operator_cert)); + +// DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); +// proposal_id = r.proposal_id; +// } + +// { +// DOCTEST_INFO("Validate vote tally"); + +// const auto readj = create_signed_request( +// read_params(proposal_id, Tables::PROPOSALS), "read", kp); + +// const auto proposal = get_proposal(frontend, proposal_id, operator_cert); + +// const auto& votes = proposal.votes; +// DOCTEST_CHECK(votes.size() == 1); + +// const auto proposer_vote = votes.find(operator_id); +// DOCTEST_CHECK(proposer_vote != votes.end()); +// DOCTEST_CHECK(proposer_vote->second == vote_for); +// } +// } + +// DOCTEST_TEST_CASE("Members passing an operator vote") +// { +// // Operator proposes a vote, but does not vote for it +// // A majority of members pass the vote +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// auto new_kp = tls::make_key_pair(); +// auto new_ca = new_kp->self_sign("CN=new node"); +// NodeInfo ni; +// ni.cert = new_ca; +// gen.add_node(ni); + +// // Operating member, as set in operator_gov.lua +// const auto operator_cert = get_cert_data(0, kp); +// const auto operator_id = gen.add_member(operator_cert, {}); + +// // Non-operating members +// std::map members; +// for (size_t i = 1; i < 4; i++) +// { +// auto cert = get_cert_data(i, kp); +// members[gen.add_member(cert, {})] = cert; +// } + +// set_whitelists(gen); +// gen.set_gov_scripts( +// lua::Interpreter().invoke(operator_gov_script_file)); +// gen.finalize(); + +// StubNodeState node; +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// size_t proposal_id; + +// const ccf::Script vote_for("return true"); +// const ccf::Script vote_against("return false"); + +// auto node_id = 0; +// { +// DOCTEST_INFO("Check node exists with status pending"); +// const auto read_values = +// create_request(read_params(node_id, Tables::NODES), "read"); +// const auto r = parse_response_body( +// frontend_process(frontend, read_values, operator_cert)); +// DOCTEST_CHECK(r.status == NodeStatus::PENDING); +// } + +// { +// DOCTEST_INFO("Operator proposes and votes against adding node"); +// Script proposal(R"xxx( +// local tables, node_id = ... +// return Calls:call("trust_node", node_id) +// )xxx"); + +// const auto propose = create_signed_request( +// Propose::In{proposal, node_id, vote_against}, "propose", kp); +// const auto r = parse_response_body( +// frontend_process(frontend, propose, operator_cert)); + +// DOCTEST_CHECK(r.state == ProposalState::OPEN); +// proposal_id = r.proposal_id; +// } + +// size_t first_voter_id = 1; +// size_t second_voter_id = 2; + +// { +// DOCTEST_INFO("First member votes for proposal"); + +// const auto vote = +// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); +// const auto r = frontend_process(frontend, vote, members[first_voter_id]); + +// check_result_state(r, ProposalState::OPEN); +// } + +// { +// DOCTEST_INFO("Second member votes for proposal"); + +// const auto vote = +// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); +// const auto r = frontend_process(frontend, vote, +// members[second_voter_id]); + +// check_result_state(r, ProposalState::ACCEPTED); +// } + +// { +// DOCTEST_INFO("Validate vote tally"); + +// const auto readj = create_signed_request( +// read_params(proposal_id, Tables::PROPOSALS), "read", kp); + +// const auto proposal = get_proposal(frontend, proposal_id, operator_cert); + +// const auto& votes = proposal.votes; +// DOCTEST_CHECK(votes.size() == 3); + +// const auto proposer_vote = votes.find(operator_id); +// DOCTEST_CHECK(proposer_vote != votes.end()); +// DOCTEST_CHECK(proposer_vote->second == vote_against); + +// const auto first_vote = votes.find(first_voter_id); +// DOCTEST_CHECK(first_vote != votes.end()); +// DOCTEST_CHECK(first_vote->second == vote_for); + +// const auto second_vote = votes.find(second_voter_id); +// DOCTEST_CHECK(second_vote != votes.end()); +// DOCTEST_CHECK(second_vote->second == vote_for); +// } +// } + +// DOCTEST_TEST_CASE("User data") +// { +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// const auto member_id = gen.add_member(member_cert, {}); +// const auto user_id = gen.add_user(user_cert); +// set_whitelists(gen); +// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); +// gen.finalize(); + +// StubNodeState node; +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// const auto read_user_info = +// create_request(read_params(user_id, Tables::USERS), "read"); + +// { +// DOCTEST_INFO("user data is initially empty"); +// const auto read_response = parse_response_body( +// frontend_process(frontend, read_user_info, member_cert)); +// DOCTEST_CHECK(read_response.user_data.is_null()); +// } + +// { +// auto user_data_object = nlohmann::json::object(); +// user_data_object["name"] = "bob"; +// user_data_object["permissions"] = {"read", "delete"}; + +// DOCTEST_INFO("user data can be set to an object"); +// Propose::In proposal; +// proposal.script = fmt::format( +// R"xxx( +// proposed_user_data = {{ +// name = "bob", +// permissions = {{"read", "delete"}} +// }} +// return Calls:call("set_user_data", {{user_id = {}, user_data = +// proposed_user_data}}) +// )xxx", +// user_id); +// const auto proposal_serialized = +// create_signed_request(proposal, "propose", kp); +// const auto propose_response = parse_response_body( +// frontend_process(frontend, proposal_serialized, member_cert)); +// DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); + +// DOCTEST_INFO("user data object can be read"); +// const auto read_response = parse_response_body( +// frontend_process(frontend, read_user_info, member_cert)); +// DOCTEST_CHECK(read_response.user_data == user_data_object); +// } + +// { +// const auto user_data_string = "ADMINISTRATOR"; + +// DOCTEST_INFO("user data can be overwritten"); +// Propose::In proposal; +// proposal.script = std::string(R"xxx( +// local tables, param = ... +// return Calls:call("set_user_data", {user_id = param.id, user_data = +// param.data}) +// )xxx"); +// proposal.parameter["id"] = user_id; +// proposal.parameter["data"] = user_data_string; +// const auto proposal_serialized = +// create_signed_request(proposal, "propose", kp); +// const auto propose_response = parse_response_body( +// frontend_process(frontend, proposal_serialized, member_cert)); +// DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); + +// DOCTEST_INFO("user data object can be read"); +// const auto response = parse_response_body( +// frontend_process(frontend, read_user_info, member_cert)); +// DOCTEST_CHECK(response.user_data == user_data_string); +// } +// } DOCTEST_TEST_CASE("Submit recovery shares") { // Setup original state - NetworkTables network; - auto node = StubNodeState(std::make_shared(network)); - MemberRpcFrontend frontend(network, node); - std::map members; + NetworkState network(ConsensusType::RAFT); + network.ledger_secrets = std::make_shared(); + network.ledger_secrets->init(); + network.encryption_key = std::make_unique( + tls::create_entropy()->random(crypto::BoxKey::KEY_SIZE)); + + ShareManager share_manager(network); + auto node = StubNodeState(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + std::map>> members; + size_t members_count = 4; size_t recovery_threshold = 2; DOCTEST_REQUIRE(recovery_threshold <= members_count); - std::map retrieved_shares; + std::map> retrieved_shares; DOCTEST_INFO("Setup state"); { Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); gen.init_values(); gen.create_service({}); @@ -1589,24 +1610,35 @@ DOCTEST_TEST_CASE("Submit recovery shares") for (size_t i = 0; i < members_count; i++) { auto cert = get_cert_data(i, kp); - members[gen.add_member(cert, {})] = cert; + auto private_encryption_key = + tls::create_entropy()->random(crypto::BoxKey::KEY_SIZE); + auto public_encryption_key = tls::PublicX25519::write( + crypto::BoxKey::public_from_private(private_encryption_key)); + members[gen.add_member(cert, public_encryption_key.raw())] = { + cert, private_encryption_key}; } gen.set_recovery_threshold(recovery_threshold); - DOCTEST_REQUIRE(node.split_ledger_secrets(gen_tx)); + share_manager.issue_shares(gen_tx); gen.finalize(); - frontend.open(); } - DOCTEST_INFO("Retrieve recovery shares"); + DOCTEST_INFO("Retrieve and decrypt recovery shares"); { const auto get_recovery_shares = create_request(nullptr, "getEncryptedRecoveryShare", HTTP_GET); for (auto const& m : members) { - retrieved_shares[m.first] = parse_response_body( - frontend_process(frontend, get_recovery_shares, m.second)); + auto encrypted_share = parse_response_body( + frontend_process(frontend, get_recovery_shares, m.second.first)); + + retrieved_shares[m.first] = crypto::Box::open( + encrypted_share.encrypted_share, + encrypted_share.nonce, + crypto::BoxKey::public_from_private( + network.encryption_key->private_raw), + m.second.second); } } @@ -1614,11 +1646,12 @@ DOCTEST_TEST_CASE("Submit recovery shares") { MemberId member_id = 0; const auto submit_recovery_share = create_request( - SubmitRecoveryShare({retrieved_shares[member_id].encrypted_share}), + SubmitRecoveryShare({retrieved_shares[member_id]}), "submitRecoveryShare"); check_error( - frontend_process(frontend, submit_recovery_share, members[member_id]), + frontend_process( + frontend, submit_recovery_share, members[member_id].first), HTTP_STATUS_FORBIDDEN); } @@ -1626,120 +1659,122 @@ DOCTEST_TEST_CASE("Submit recovery shares") { Store::Tx tx; GenesisGenerator g(network, tx); - DOCTEST_REQUIRE(g.service_wait_for_shares()); - g.finalize(); } + DOCTEST_INFO( + "Threshold cannot be changed while service is waiting for shares"); + { + Store::Tx tx; + GenesisGenerator g(network, tx); + DOCTEST_REQUIRE_FALSE(g.set_recovery_threshold(recovery_threshold)); + } + DOCTEST_INFO("Submit recovery shares"); { - size_t member_count = 0; + size_t submitted_shares_count = 0; for (auto const& m : members) { const auto submit_recovery_share = create_request( - SubmitRecoveryShare({retrieved_shares[m.first].encrypted_share}), + SubmitRecoveryShare({retrieved_shares[m.first]}), "submitRecoveryShare"); auto ret = parse_response_body( - frontend_process(frontend, submit_recovery_share, m.second)); + frontend_process(frontend, submit_recovery_share, m.second.first)); - member_count++; + submitted_shares_count++; - // Share submission should only complete when the recovery threshold has - // been reached - if (member_count < recovery_threshold) - { - DOCTEST_REQUIRE(!ret); - } - else + // Share submission should only complete when the recovery threshold + // has been reached + DOCTEST_REQUIRE(ret == (submitted_shares_count >= recovery_threshold)); + if (ret) { - DOCTEST_REQUIRE(ret); break; } } } } -DOCTEST_TEST_CASE("Maximum number of active members") -{ - logger::config::level() = logger::INFO; - - NetworkTables network; - network.tables->set_encryptor(encryptor); - auto node = StubNodeState(std::make_shared(network)); - MemberRpcFrontend frontend(network, node); - frontend.open(); - - DOCTEST_INFO("Service is opening"); - { - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - - for (size_t i = 0; i < max_active_members_count + 1; i++) - { - auto cert = get_cert_data(i, kp); - if (i == max_active_members_count) - { - DOCTEST_REQUIRE_THROWS_AS_MESSAGE( - gen.add_member(cert, {}), - std::logic_error, - fmt::format( - "No more than {} active members are allowed", - max_active_members_count)); - } - else - { - gen.add_member(cert, {}); - } - } - } - - DOCTEST_INFO("Service is open"); - { - std::map members; - - Store::Tx gen_tx; - GenesisGenerator gen(network, gen_tx); - gen.init_values(); - gen.create_service({}); - gen.open_service(); - gen.set_recovery_threshold(1); - - // Service is open so members are added as ACCEPTED - for (size_t i = 0; i < max_active_members_count + 1; i++) - { - auto cert = get_cert_data(i, kp); - members[gen.add_member(cert, {})] = cert; - } - gen.finalize(); - - for (auto const& m : members) - { - const auto state_digest_req = - create_request(nullptr, "updateAckStateDigest"); - const auto ack = parse_response_body( - frontend_process(frontend, state_digest_req, m.second)); - - StateDigest params; - params.state_digest = ack.state_digest; - const auto ack_req = create_signed_request(params, "ack", kp); - const auto resp = frontend_process(frontend, ack_req, m.second); - - if (m.first >= max_active_members_count) - { - DOCTEST_CHECK(resp.status == HTTP_STATUS_FORBIDDEN); - } - else - { - DOCTEST_CHECK(resp.status == HTTP_STATUS_OK); - DOCTEST_CHECK(parse_response_body(resp)); - } - } - } -} +// DOCTEST_TEST_CASE("Maximum number of active members") +// { +// logger::config::level() = logger::INFO; + +// NetworkTables network; +// network.tables->set_encryptor(encryptor); +// auto node = StubNodeState(std::make_shared(network)); +// MemberRpcFrontend frontend(network, node); +// frontend.open(); + +// DOCTEST_INFO("Service is opening"); +// { +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); + +// for (size_t i = 0; i < max_active_members_count + 1; i++) +// { +// auto cert = get_cert_data(i, kp); +// if (i == max_active_members_count) +// { +// DOCTEST_REQUIRE_THROWS_AS_MESSAGE( +// gen.add_member(cert, {}), +// std::logic_error, +// fmt::format( +// "No more than {} active members are allowed", +// max_active_members_count)); +// } +// else +// { +// gen.add_member(cert, {}); +// } +// } +// } + +// DOCTEST_INFO("Service is open"); +// { +// std::map members; + +// Store::Tx gen_tx; +// GenesisGenerator gen(network, gen_tx); +// gen.init_values(); +// gen.create_service({}); +// gen.open_service(); +// gen.set_recovery_threshold(1); + +// // Service is open so members are added as ACCEPTED +// for (size_t i = 0; i < max_active_members_count + 1; i++) +// { +// auto cert = get_cert_data(i, kp); +// members[gen.add_member(cert, {})] = cert; +// } +// gen.finalize(); + +// for (auto const& m : members) +// { +// const auto state_digest_req = +// create_request(nullptr, "updateAckStateDigest"); +// const auto ack = parse_response_body( +// frontend_process(frontend, state_digest_req, m.second)); + +// StateDigest params; +// params.state_digest = ack.state_digest; +// const auto ack_req = create_signed_request(params, "ack", kp); +// const auto resp = frontend_process(frontend, ack_req, m.second); + +// if (m.first >= max_active_members_count) +// { +// DOCTEST_CHECK(resp.status == HTTP_STATUS_FORBIDDEN); +// } +// else +// { +// DOCTEST_CHECK(resp.status == HTTP_STATUS_OK); +// DOCTEST_CHECK(parse_response_body(resp)); +// } +// } +// } +// } // We need an explicit main to initialize kremlib and EverCrypt int main(int argc, char** argv) diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h index 7897e703beb6..8589d61d1245 100644 --- a/src/node/rpc/test/node_stub.h +++ b/src/node/rpc/test/node_stub.h @@ -3,7 +3,7 @@ #pragma once #include "node/rpc/node_interface.h" -#include "node/secret_share.h" +#include "node/share_manager.h" namespace ccf { @@ -11,17 +11,10 @@ namespace ccf { private: bool is_public = false; - std::shared_ptr network; - - NetworkState network_state; - ShareManager share_manager; + ShareManager& share_manager; public: - StubNodeState(std::shared_ptr network_ = nullptr) : - network(network_), - network_state(ConsensusType::RAFT), // TODO: Remove - share_manager(network_state) // TODO: Remove - {} + StubNodeState(ShareManager& share_manager) : share_manager(share_manager) {} bool accept_recovery(Store::Tx& tx) override { @@ -69,50 +62,9 @@ namespace ccf const std::optional>& filter) override {} - bool split_ledger_secrets(Store::Tx& tx) override - { - auto [members_view, shares_view] = - tx.get_view(network->members, network->shares); - SecretSharing::SplitSecret secret_to_split = {}; - - GenesisGenerator g(*network.get(), tx); - auto active_member_count = g.get_active_members_count(); - size_t threshold = g.get_recovery_threshold(); - - auto shares = - SecretSharing::split(secret_to_split, active_member_count, threshold); - - // Here, shares are not encrypted but recorded in plain text - EncryptedSharesMap recorded_shares; - MemberId member_id = 0; - for (auto const& s : shares) - { - auto share_raw = std::vector(s.begin(), s.end()); - recorded_shares[member_id] = {{}, share_raw}; - member_id++; - } - g.add_key_share_info({{}, {}, recorded_shares}); - - return true; - } - void restore_ledger_secrets(Store::Tx& tx) override { - auto submitted_shares = tx.get_view(network->submitted_shares)->get(0); - if (!submitted_shares.has_value()) - { - throw std::logic_error("Could not find submitted shares"); - } - - std::vector shares; - for (auto const& s : submitted_shares.value()) - { - SecretSharing::Share share; - std::copy_n( - s.second.begin(), SecretSharing::SHARE_LENGTH, share.begin()); - shares.emplace_back(share); - } - SecretSharing::combine(shares, shares.size()); + share_manager.restore_recovery_shares_info(tx, {}); } kv::Version get_last_recovered_commit_idx() override @@ -120,12 +72,6 @@ namespace ccf return kv::NoVersion; } - // TODO: Super bad hack for now. Please fix this!! - ShareManager& get_share_manager() override - { - return share_manager; - } - NodeId get_node_id() const override { return 0; From bfa1a3cb8c840bbc1afe84481f170e46e12b0678 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 11:22:15 +0100 Subject: [PATCH 13/33] Membervoting test works again --- src/node/genesis_gen.h | 6 + src/node/rpc/member_frontend.h | 2 +- src/node/rpc/test/member_voting_test.cpp | 2982 +++++++++++----------- 3 files changed, 1512 insertions(+), 1478 deletions(-) diff --git a/src/node/genesis_gen.h b/src/node/genesis_gen.h index 97db84aefdf7..87ea854878c2 100644 --- a/src/node/genesis_gen.h +++ b/src/node/genesis_gen.h @@ -414,6 +414,12 @@ namespace ccf return false; } + if (threshold == 0) + { + LOG_FAIL_FMT("Recovery threshold cannot be set to 0"); + return false; + } + auto active_members_count = get_active_members_count(); if (threshold > active_members_count) { diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 58636640c97e..1b8ef3967694 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -816,7 +816,7 @@ namespace ccf { return make_error( HTTP_STATUS_INTERNAL_SERVER_ERROR, - fmt::format("Error issuing new recovery shares {}", e.what())); + fmt::format("Error issuing new recovery shares {}: ", e.what())); } } return make_success(true); diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 85f46c014d1c..4fd87718d5a5 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -182,1404 +182,1424 @@ std::vector get_cert_data(uint64_t member_id, tls::KeyPairPtr& kp_mem) return kp_mem->self_sign("CN=new member" + to_string(member_id)); } -// auto init_frontend( -// NetworkTables& network, -// GenesisGenerator& gen, -// StubNodeState& node, -// const int n_members, -// std::vector>& member_certs) -// { -// // create members -// for (uint8_t i = 0; i < n_members; i++) -// { -// member_certs.push_back(get_cert_data(i, kp)); -// gen.add_member(member_certs.back(), {}); -// } - -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); -// gen.finalize(); - -// return MemberRpcFrontend(network, node); -// } - -// DOCTEST_TEST_CASE("Member query/read") -// { -// // initialize the network state -// NetworkTables network; -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node; -// MemberRpcFrontend frontend(network, node); -// frontend.open(); -// const auto member_id = gen.add_member(member_cert, {}); -// gen.finalize(); - -// const enclave::SessionContext member_session( -// enclave::InvalidSessionId, member_cert); - -// // put value to read -// constexpr auto key = 123; -// constexpr auto value = 456; -// Store::Tx tx; -// tx.get_view(network.values)->put(key, value); -// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - -// static constexpr auto query = R"xxx( -// local tables = ... -// return tables["ccf.values"]:get(123) -// )xxx"; - -// DOCTEST_SUBCASE("Query: bytecode/script allowed access") -// { -// // set member ACL so that the VALUES table is accessible -// Store::Tx tx; -// tx.get_view(network.whitelists) -// ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); -// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - -// bool compile = true; -// do -// { -// const auto req = create_request(query_params(query, compile), "query"); -// const auto r = frontend_process(frontend, req, member_cert); -// const auto result = parse_response_body(r); -// DOCTEST_CHECK(result == value); -// compile = !compile; -// } while (!compile); -// } - -// DOCTEST_SUBCASE("Query: table not in ACL") -// { -// // set member ACL so that no table is accessible -// Store::Tx tx; -// tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); -// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - -// auto req = create_request(query_params(query, true), "query"); -// const auto response = frontend_process(frontend, req, member_cert); - -// check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); -// } - -// DOCTEST_SUBCASE("Read: allowed access, key exists") -// { -// Store::Tx tx; -// tx.get_view(network.whitelists) -// ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); -// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - -// auto read_call = -// create_request(read_params(key, Tables::VALUES), "read"); -// const auto r = frontend_process(frontend, read_call, member_cert); -// const auto result = parse_response_body(r); -// DOCTEST_CHECK(result == value); -// } - -// DOCTEST_SUBCASE("Read: allowed access, key doesn't exist") -// { -// constexpr auto wrong_key = 321; -// Store::Tx tx; -// tx.get_view(network.whitelists) -// ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); -// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - -// auto read_call = -// create_request(read_params(wrong_key, Tables::VALUES), "read"); -// const auto response = frontend_process(frontend, read_call, member_cert); - -// check_error(response, HTTP_STATUS_BAD_REQUEST); -// } - -// DOCTEST_SUBCASE("Read: access not allowed") -// { -// Store::Tx tx; -// tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); -// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); - -// auto read_call = -// create_request(read_params(key, Tables::VALUES), "read"); -// const auto response = frontend_process(frontend, read_call, member_cert); - -// check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); -// } -// } - -// DOCTEST_TEST_CASE("Proposer ballot") -// { -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); - -// const auto proposer_cert = get_cert_data(0, kp); -// const auto proposer_id = gen.add_member(proposer_cert, {}); -// const auto voter_cert = get_cert_data(1, kp); -// const auto voter_id = gen.add_member(voter_cert, {}); - -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); -// gen.finalize(); - -// StubNodeState node; -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// size_t proposal_id; - -// const ccf::Script vote_for("return true"); -// const ccf::Script vote_against("return false"); -// { -// DOCTEST_INFO("Propose, initially voting against"); - -// const auto proposed_member = get_cert_data(2, kp); - -// Propose::In proposal; -// proposal.script = std::string(R"xxx( -// tables, member_info = ... -// return Calls:call("new_member", member_info) -// )xxx"); -// proposal.parameter["cert"] = proposed_member; -// proposal.parameter["keyshare"] = dummy_key_share; -// proposal.ballot = vote_against; -// const auto propose = create_signed_request(proposal, "propose", kp); -// const auto r = frontend_process(frontend, propose, proposer_cert); - -// // the proposal should be accepted, but not succeed immediately -// const auto result = parse_response_body(r); -// DOCTEST_CHECK(result.state == ProposalState::OPEN); - -// proposal_id = result.proposal_id; -// } - -// { -// DOCTEST_INFO("Second member votes for proposal"); - -// const auto vote = -// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); -// const auto r = frontend_process(frontend, vote, voter_cert); - -// // The vote should not yet succeed -// check_result_state(r, ProposalState::OPEN); -// } - -// { -// DOCTEST_INFO("Read current votes"); - -// const auto proposal_result = -// get_proposal(frontend, proposal_id, proposer_cert); - -// const auto& votes = proposal_result.votes; -// DOCTEST_CHECK(votes.size() == 2); - -// const auto proposer_vote = votes.find(proposer_id); -// DOCTEST_CHECK(proposer_vote != votes.end()); -// DOCTEST_CHECK(proposer_vote->second == vote_against); - -// const auto voter_vote = votes.find(voter_id); -// DOCTEST_CHECK(voter_vote != votes.end()); -// DOCTEST_CHECK(voter_vote->second == vote_for); -// } - -// { -// DOCTEST_INFO("Proposer votes for"); - -// const auto vote = -// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); -// const auto r = frontend_process(frontend, vote, proposer_cert); - -// // The vote should now succeed -// check_result_state(r, ProposalState::ACCEPTED); -// } -// } - -// struct NewMember -// { -// MemberId id; -// tls::KeyPairPtr kp = tls::make_key_pair(); -// Cert cert; -// }; - -// DOCTEST_TEST_CASE("Add new members until there are 7 then reject") -// { -// logger::config::level() = logger::INFO; - -// constexpr auto initial_members = 3; -// constexpr auto n_new_members = 7; -// constexpr auto max_members = 8; -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node(std::make_shared(network)); -// // add three initial active members -// // the proposer -// auto proposer_id = gen.add_member(member_cert, {}); - -// // the voters -// const auto voter_a_cert = get_cert_data(1, kp); -// auto voter_a = gen.add_member(voter_a_cert, {}); -// const auto voter_b_cert = get_cert_data(2, kp); -// auto voter_b = gen.add_member(voter_b_cert, {}); - -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); -// gen.set_recovery_threshold(1); -// gen.open_service(); -// gen.finalize(); -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// vector new_members(n_new_members); - -// auto i = 0ul; -// for (auto& new_member : new_members) -// { -// const auto proposal_id = i; -// new_member.id = initial_members + i++; - -// // new member certificate -// auto cert_pem = -// new_member.kp->self_sign(fmt::format("CN=new member{}", -// new_member.id)); -// auto keyshare = dummy_key_share; -// auto v = tls::make_verifier(cert_pem); -// const auto _cert = v->raw(); -// new_member.cert = {_cert->raw.p, _cert->raw.p + _cert->raw.len}; - -// // check new_member id does not work before member is added -// const auto read_next_req = create_request( -// read_params(ValueIds::NEXT_MEMBER_ID, Tables::VALUES), "read"); -// const auto r = frontend_process(frontend, read_next_req, -// new_member.cert); check_error(r, HTTP_STATUS_FORBIDDEN); - -// // propose new member, as proposer -// Propose::In proposal; -// proposal.script = std::string(R"xxx( -// tables, member_info = ... -// return Calls:call("new_member", member_info) -// )xxx"); -// proposal.parameter["cert"] = cert_pem; -// proposal.parameter["keyshare"] = keyshare; - -// const auto propose = create_signed_request(proposal, "propose", kp); - -// { -// const auto r = frontend_process(frontend, propose, member_cert); -// const auto result = parse_response_body(r); - -// // the proposal should be accepted, but not succeed immediately -// DOCTEST_CHECK(result.proposal_id == proposal_id); -// DOCTEST_CHECK(result.state == ProposalState::OPEN); -// } - -// // read initial proposal, as second member -// const Proposal initial_read = -// get_proposal(frontend, proposal_id, voter_a_cert); -// DOCTEST_CHECK(initial_read.proposer == proposer_id); -// DOCTEST_CHECK(initial_read.script == proposal.script); -// DOCTEST_CHECK(initial_read.parameter == proposal.parameter); - -// // vote as second member -// Script vote_ballot(fmt::format( -// R"xxx( -// local tables, calls = ... -// local n = 0 -// tables["ccf.members"]:foreach( function(k, v) n = n + 1 end ) -// if n < {} then -// return true -// else -// return false -// end -// )xxx", -// max_members)); - -// const auto vote = -// create_signed_request(Vote{proposal_id, vote_ballot}, "vote", kp); - -// { -// const auto r = frontend_process(frontend, vote, voter_a_cert); -// const auto result = parse_response_body(r); - -// if (new_member.id < max_members) -// { -// // vote should succeed -// DOCTEST_CHECK(result.state == ProposalState::ACCEPTED); -// // check that member with the new new_member cert can make RPCs now -// DOCTEST_CHECK( -// parse_response_body(frontend_process( -// frontend, read_next_req, new_member.cert)) == new_member.id + 1); - -// // successful proposals are removed from the kv, so we can't confirm -// // their final state -// } -// else -// { -// // vote should not succeed -// DOCTEST_CHECK(result.state == ProposalState::OPEN); -// // check that member with the new new_member cert can make RPCs now -// check_error( -// frontend_process(frontend, read_next_req, new_member.cert), -// HTTP_STATUS_FORBIDDEN); - -// // re-read proposal, as second member -// const Proposal final_read = -// get_proposal(frontend, proposal_id, voter_a_cert); -// DOCTEST_CHECK(final_read.proposer == proposer_id); -// DOCTEST_CHECK(final_read.script == proposal.script); -// DOCTEST_CHECK(final_read.parameter == proposal.parameter); - -// const auto my_vote = final_read.votes.find(voter_a); -// DOCTEST_CHECK(my_vote != final_read.votes.end()); -// DOCTEST_CHECK(my_vote->second == vote_ballot); -// } -// } -// } - -// DOCTEST_SUBCASE("ACK from newly added members") -// { -// // iterate over all new_members, except for the last one -// for (auto new_member = new_members.cbegin(); new_member != -// new_members.cend() - (initial_members + n_new_members - -// max_members); new_member++) -// { -// // (1) read ack entry -// const auto read_state_digest_req = create_request( -// read_params(new_member->id, Tables::MEMBER_ACKS), "read"); -// const auto ack0 = parse_response_body( -// frontend_process(frontend, read_state_digest_req, new_member->cert)); -// DOCTEST_REQUIRE(std::all_of( -// ack0.state_digest.begin(), ack0.state_digest.end(), [](uint8_t i) { -// return i == 0; -// })); - -// { -// // make sure that there is a signature in the signatures table since -// // ack's depend on that -// Store::Tx tx; -// auto sig_view = tx.get_view(network.signatures); -// Signature sig_value; -// sig_view->put(0, sig_value); -// DOCTEST_REQUIRE(tx.commit() == kv::CommitSuccess::OK); -// } - -// // (2) ask for a fresher digest of state -// const auto freshen_state_digest_req = -// create_request(nullptr, "updateAckStateDigest"); -// const auto freshen_state_digest = parse_response_body( -// frontend_process(frontend, freshen_state_digest_req, -// new_member->cert)); -// DOCTEST_CHECK(freshen_state_digest.state_digest != ack0.state_digest); - -// // (3) read ack entry again and check that the state digest has changed -// const auto ack1 = parse_response_body( -// frontend_process(frontend, read_state_digest_req, new_member->cert)); -// DOCTEST_CHECK(ack0.state_digest != ack1.state_digest); -// DOCTEST_CHECK(freshen_state_digest.state_digest == ack1.state_digest); - -// // (4) sign stale state and send it -// StateDigest params; -// params.state_digest = ack0.state_digest; -// const auto send_stale_sig_req = -// create_signed_request(params, "ack", new_member->kp); -// check_error( -// frontend_process(frontend, send_stale_sig_req, new_member->cert), -// HTTP_STATUS_BAD_REQUEST); - -// // (5) sign new state digest and send it -// params.state_digest = ack1.state_digest; -// const auto send_good_sig_req = -// create_signed_request(params, "ack", new_member->kp); -// const auto good_response = -// frontend_process(frontend, send_good_sig_req, new_member->cert); -// DOCTEST_CHECK(good_response.status == HTTP_STATUS_OK); -// DOCTEST_CHECK(parse_response_body(good_response)); - -// // (6) read own member status -// const auto read_status_req = -// create_request(read_params(new_member->id, Tables::MEMBERS), "read"); -// const auto mi = parse_response_body( -// frontend_process(frontend, read_status_req, new_member->cert)); -// DOCTEST_CHECK(mi.status == MemberStatus::ACTIVE); -// } -// } -// } - -// DOCTEST_TEST_CASE("Accept node") -// { -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node; -// auto new_kp = tls::make_key_pair(); - -// const auto member_0_cert = get_cert_data(0, new_kp); -// const auto member_1_cert = get_cert_data(1, kp); -// const auto member_0 = gen.add_member(member_0_cert, {}); -// const auto member_1 = gen.add_member(member_1_cert, {}); - -// // node to be tested -// // new node certificate -// auto new_ca = new_kp->self_sign("CN=new node"); -// NodeInfo ni; -// ni.cert = new_ca; -// gen.add_node(ni); -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); -// gen.finalize(); -// MemberRpcFrontend frontend(network, node); -// frontend.open(); -// auto node_id = 0; - -// // check node exists with status pending -// { -// auto read_values = -// create_request(read_params(node_id, Tables::NODES), "read"); -// const auto r = parse_response_body( -// frontend_process(frontend, read_values, member_0_cert)); - -// DOCTEST_CHECK(r.status == NodeStatus::PENDING); -// } - -// // m0 proposes adding new node -// { -// Script proposal(R"xxx( -// local tables, node_id = ... -// return Calls:call("trust_node", node_id) -// )xxx"); -// const auto propose = -// create_signed_request(Propose::In{proposal, node_id}, "propose", -// new_kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_0_cert)); - -// DOCTEST_CHECK(r.state == ProposalState::OPEN); -// DOCTEST_CHECK(r.proposal_id == 0); -// } - -// // m1 votes for accepting a single new node -// { -// Script vote_ballot(R"xxx( -// local tables, calls = ... -// return #calls == 1 and calls[1].func == "trust_node" -// )xxx"); -// const auto vote = create_signed_request(Vote{0, vote_ballot}, "vote", -// kp); - -// check_result_state( -// frontend_process(frontend, vote, member_1_cert), -// ProposalState::ACCEPTED); -// } - -// // check node exists with status pending -// { -// const auto read_values = -// create_request(read_params(node_id, Tables::NODES), "read"); -// const auto r = parse_response_body( -// frontend_process(frontend, read_values, member_0_cert)); -// DOCTEST_CHECK(r.status == NodeStatus::TRUSTED); -// } - -// // m0 proposes retire node -// { -// Script proposal(R"xxx( -// local tables, node_id = ... -// return Calls:call("retire_node", node_id) -// )xxx"); -// const auto propose = -// create_signed_request(Propose::In{proposal, node_id}, "propose", -// new_kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_0_cert)); - -// DOCTEST_CHECK(r.state == ProposalState::OPEN); -// DOCTEST_CHECK(r.proposal_id == 1); -// } - -// // m1 votes for retiring node -// { -// const Script vote_ballot("return true"); -// const auto vote = create_signed_request(Vote{1, vote_ballot}, "vote", -// kp); check_result_state( -// frontend_process(frontend, vote, member_1_cert), -// ProposalState::ACCEPTED); -// } - -// // check that node exists with status retired -// { -// auto read_values = -// create_request(read_params(node_id, Tables::NODES), "read"); -// const auto r = parse_response_body( -// frontend_process(frontend, read_values, member_0_cert)); -// DOCTEST_CHECK(r.status == NodeStatus::RETIRED); -// } - -// // check that retired node cannot be trusted -// { -// Script proposal(R"xxx( -// local tables, node_id = ... -// return Calls:call("trust_node", node_id) -// )xxx"); -// const auto propose = -// create_signed_request(Propose::In{proposal, node_id}, "propose", -// new_kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_0_cert)); - -// const Script vote_ballot("return true"); -// const auto vote = -// create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); -// check_result_state( -// frontend_process(frontend, vote, member_1_cert), -// ProposalState::FAILED); -// } - -// // check that retired node cannot be retired again -// { -// Script proposal(R"xxx( -// local tables, node_id = ... -// return Calls:call("retire_node", node_id) -// )xxx"); -// const auto propose = -// create_signed_request(Propose::In{proposal, node_id}, "propose", -// new_kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_0_cert)); - -// const Script vote_ballot("return true"); -// const auto vote = -// create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); -// check_result_state( -// frontend_process(frontend, vote, member_1_cert), -// ProposalState::FAILED); -// } -// } - -// ProposalInfo test_raw_writes( -// NetworkTables& network, -// GenesisGenerator& gen, -// StubNodeState& node, -// Propose::In proposal, -// const int n_members = 1, -// const int pro_votes = 1, -// bool explicit_proposer_vote = false) -// { -// std::vector> member_certs; -// auto frontend = init_frontend(network, gen, node, n_members, member_certs); -// frontend.open(); - -// // check values before -// { -// Store::Tx tx; -// auto next_member_id_r = -// tx.get_view(network.values)->get(ValueIds::NEXT_MEMBER_ID); -// DOCTEST_CHECK(next_member_id_r); -// DOCTEST_CHECK(*next_member_id_r == n_members); -// } - -// // propose -// const auto proposal_id = 0ul; -// { -// const uint8_t proposer_id = 0; -// const auto propose = create_signed_request(proposal, "propose", kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_certs[0])); - -// const auto expected_state = -// (n_members == 1) ? ProposalState::ACCEPTED : ProposalState::OPEN; -// DOCTEST_CHECK(r.state == expected_state); -// DOCTEST_CHECK(r.proposal_id == proposal_id); -// if (r.state == ProposalState::ACCEPTED) -// return r; -// } - -// // con votes -// for (int i = n_members - 1; i >= pro_votes; i--) -// { -// const Script vote("return false"); -// const auto vote_serialized = -// create_signed_request(Vote{proposal_id, vote}, "vote", kp); - -// check_result_state( -// frontend_process(frontend, vote_serialized, member_certs[i]), -// ProposalState::OPEN); -// } - -// // pro votes (proposer also votes) -// ProposalInfo info = {}; -// for (uint8_t i = explicit_proposer_vote ? 0 : 1; i < pro_votes; i++) -// { -// const Script vote("return true"); -// const auto vote_serialized = -// create_signed_request(Vote{proposal_id, vote}, "vote", kp); -// if (info.state == ProposalState::OPEN) -// { -// info = parse_response_body( -// frontend_process(frontend, vote_serialized, member_certs[i])); -// } -// else -// { -// // proposal has been accepted - additional votes return an error -// check_error( -// frontend_process(frontend, vote_serialized, member_certs[i]), -// HTTP_STATUS_BAD_REQUEST); -// } -// } -// return info; -// } - -// DOCTEST_TEST_CASE("Propose raw writes") -// { -// logger::config::level() = logger::INFO; -// DOCTEST_SUBCASE("insensitive tables") -// { -// const auto n_members = 3; -// for (int pro_votes = 0; pro_votes <= n_members; pro_votes++) -// { -// const bool should_succeed = pro_votes > n_members / 2; -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node; -// nlohmann::json recovery_threshold = 4; - -// Store::Tx tx_before; -// auto configuration = tx_before.get_view(network.config)->get(0); -// DOCTEST_REQUIRE_FALSE(configuration.has_value()); - -// const auto expected_state = -// should_succeed ? ProposalState::ACCEPTED : ProposalState::OPEN; -// const auto proposal_info = test_raw_writes( -// network, -// gen, -// node, -// {R"xxx( -// local tables, recovery_threshold = ... -// local p = Puts:new() -// p:put("ccf.config", 0, {recovery_threshold = recovery_threshold}) -// return Calls:call("raw_puts", p) -// )xxx"s, -// 4}, -// n_members, -// pro_votes); -// DOCTEST_CHECK(proposal_info.state == expected_state); -// if (!should_succeed) -// continue; - -// // check results -// Store::Tx tx_after; -// configuration = tx_after.get_view(network.config)->get(0); -// DOCTEST_CHECK(configuration.has_value()); -// DOCTEST_CHECK(configuration->recovery_threshold == recovery_threshold); -// } -// } - -// DOCTEST_SUBCASE("sensitive tables") -// { -// // propose changes to sensitive tables; changes must only be accepted -// // unanimously create new network for each case -// const auto sensitive_tables = {Tables::WHITELISTS, Tables::GOV_SCRIPTS}; -// const auto n_members = 3; -// // let proposer vote/not vote -// for (const auto proposer_vote : {true, false}) -// { -// for (int pro_votes = 0; pro_votes < n_members; pro_votes++) -// { -// for (const auto& sensitive_table : sensitive_tables) -// { -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node; - -// const auto sensitive_put = -// "return Calls:call('raw_puts', Puts:put('"s + sensitive_table + -// "', 9, {'aaa'}))"s; -// const auto expected_state = (n_members == pro_votes) ? -// ProposalState::ACCEPTED : -// ProposalState::OPEN; -// const auto proposal_info = test_raw_writes( -// network, -// gen, -// node, -// {sensitive_put}, -// n_members, -// pro_votes, -// proposer_vote); -// DOCTEST_CHECK(proposal_info.state == expected_state); -// } -// } -// } -// } -// } - -// DOCTEST_TEST_CASE("Remove proposal") -// { -// NewMember caller; -// auto cert = caller.kp->self_sign("CN=new member"); -// auto v = tls::make_verifier(cert); -// caller.cert = v->der_cert_data(); - -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); - -// StubNodeState node; -// gen.add_member(member_cert, {}); -// gen.add_member(cert, {}); -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); -// gen.finalize(); -// MemberRpcFrontend frontend(network, node); -// frontend.open(); -// auto proposal_id = 0; -// auto wrong_proposal_id = 1; -// ccf::Script proposal_script(R"xxx( -// local tables, param = ... -// return {} -// )xxx"); - -// // check that the proposal doesn't exist -// { -// Store::Tx tx; -// auto proposal = tx.get_view(network.proposals)->get(proposal_id); -// DOCTEST_CHECK(!proposal); -// } - -// { -// const auto propose = -// create_signed_request(Propose::In{proposal_script, 0}, "propose", kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_cert)); - -// DOCTEST_CHECK(r.proposal_id == proposal_id); -// DOCTEST_CHECK(r.state == ProposalState::OPEN); -// } - -// // check that the proposal is there -// { -// Store::Tx tx; -// auto proposal = tx.get_view(network.proposals)->get(proposal_id); -// DOCTEST_CHECK(proposal); -// DOCTEST_CHECK(proposal->state == ProposalState::OPEN); -// DOCTEST_CHECK( -// proposal->script.text.value() == proposal_script.text.value()); -// } - -// DOCTEST_SUBCASE("Attempt withdraw proposal with non existing id") -// { -// json param; -// param["id"] = wrong_proposal_id; -// const auto withdraw = create_signed_request(param, "withdraw", kp); - -// check_error( -// frontend_process(frontend, withdraw, member_cert), -// HTTP_STATUS_BAD_REQUEST); -// } - -// DOCTEST_SUBCASE("Attempt withdraw proposal that you didn't propose") -// { -// json param; -// param["id"] = proposal_id; -// const auto withdraw = create_signed_request(param, "withdraw", -// caller.kp); - -// check_error( -// frontend_process(frontend, withdraw, cert), HTTP_STATUS_FORBIDDEN); -// } - -// DOCTEST_SUBCASE("Successfully withdraw proposal") -// { -// json param; -// param["id"] = proposal_id; -// const auto withdraw = create_signed_request(param, "withdraw", kp); - -// check_result_state( -// frontend_process(frontend, withdraw, member_cert), -// ProposalState::WITHDRAWN); - -// // check that the proposal is now withdrawn -// { -// Store::Tx tx; -// auto proposal = tx.get_view(network.proposals)->get(proposal_id); -// DOCTEST_CHECK(proposal.has_value()); -// DOCTEST_CHECK(proposal->state == ProposalState::WITHDRAWN); -// } -// } -// } - -// DOCTEST_TEST_CASE("Complete proposal after initial rejection") -// { -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node; -// std::vector> member_certs; -// auto frontend = init_frontend(network, gen, node, 3, member_certs); -// frontend.open(); - -// { -// DOCTEST_INFO("Propose"); -// const auto proposal = -// "return Calls:call('raw_puts', Puts:put('ccf.values', 999, 999))"s; -// const auto propose = -// create_signed_request(Propose::In{proposal}, "propose", kp); - -// Store::Tx tx; -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_certs[0])); -// DOCTEST_CHECK(r.state == ProposalState::OPEN); -// } - -// { -// DOCTEST_INFO("Vote that rejects initially"); -// const Script vote(R"xxx( -// local tables = ... -// return tables["ccf.values"]:get(123) == 123 -// )xxx"); -// const auto vote_serialized = -// create_signed_request(Vote{0, vote}, "vote", kp); - -// check_result_state( -// frontend_process(frontend, vote_serialized, member_certs[1]), -// ProposalState::OPEN); -// } - -// { -// DOCTEST_INFO("Try to complete"); -// const auto complete = -// create_signed_request(ProposalAction{0}, "complete", kp); - -// check_result_state( -// frontend_process(frontend, complete, member_certs[1]), -// ProposalState::OPEN); -// } - -// { -// DOCTEST_INFO("Put value that makes vote agree"); -// Store::Tx tx; -// tx.get_view(network.values)->put(123, 123); -// DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); -// } - -// { -// DOCTEST_INFO("Try again to complete"); -// const auto complete = -// create_signed_request(ProposalAction{0}, "complete", kp); - -// check_result_state( -// frontend_process(frontend, complete, member_certs[1]), -// ProposalState::ACCEPTED); -// } -// } - -// DOCTEST_TEST_CASE("Vetoed proposal gets rejected") -// { -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node; -// const auto voter_a_cert = get_cert_data(1, kp); -// auto voter_a = gen.add_member(voter_a_cert, {}); -// const auto voter_b_cert = get_cert_data(2, kp); -// auto voter_b = gen.add_member(voter_b_cert, {}); -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_veto_script_file)); -// gen.finalize(); -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// Script proposal(R"xxx( -// tables, user_cert = ... -// return Calls:call("new_user", user_cert) -// )xxx"); - -// const vector user_cert = kp->self_sign("CN=new user"); -// const auto propose = -// create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); - -// const auto r = parse_response_body( -// frontend_process(frontend, propose, voter_a_cert)); -// DOCTEST_CHECK(r.state == ProposalState::OPEN); -// DOCTEST_CHECK(r.proposal_id == 0); - -// const ccf::Script vote_against("return false"); -// { -// DOCTEST_INFO("Member vetoes proposal"); - -// const auto vote = create_signed_request(Vote{0, vote_against}, "vote", -// kp); const auto r = frontend_process(frontend, vote, voter_b_cert); - -// check_result_state(r, ProposalState::REJECTED); -// } - -// { -// DOCTEST_INFO("Check proposal was rejected"); - -// const auto proposal = get_proposal(frontend, 0, voter_a_cert); - -// DOCTEST_CHECK(proposal.state == ProposalState::REJECTED); -// } -// } - -// DOCTEST_TEST_CASE("Add user via proposed call") -// { -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// StubNodeState node; -// const auto member_cert = get_cert_data(0, kp); -// gen.add_member(member_cert, {}); -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); -// gen.finalize(); -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// Script proposal(R"xxx( -// tables, user_cert = ... -// return Calls:call("new_user", user_cert) -// )xxx"); - -// const vector user_cert = kp->self_sign("CN=new user"); -// const auto propose = -// create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); - -// const auto r = parse_response_body( -// frontend_process(frontend, propose, member_cert)); -// DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); -// DOCTEST_CHECK(r.proposal_id == 0); - -// Store::Tx tx1; -// const auto uid = tx1.get_view(network.values)->get(ValueIds::NEXT_USER_ID); -// DOCTEST_CHECK(uid); -// DOCTEST_CHECK(*uid == 1); -// const auto uid1 = tx1.get_view(network.user_certs) -// ->get(tls::make_verifier(user_cert)->der_cert_data()); -// DOCTEST_CHECK(uid1); -// DOCTEST_CHECK(*uid1 == 0); -// } - -// DOCTEST_TEST_CASE("Passing members ballot with operator") -// { -// // Members pass a ballot with a constitution that includes an operator -// // Operator votes, but is _not_ taken into consideration -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); - -// // Operating member, as set in operator_gov.lua -// const auto operator_cert = get_cert_data(0, kp); -// const auto operator_id = gen.add_member(operator_cert, {}); - -// // Non-operating members -// std::map members; -// for (size_t i = 1; i < 4; i++) -// { -// auto cert = get_cert_data(i, kp); -// members[gen.add_member(cert, {})] = cert; -// } - -// set_whitelists(gen); -// gen.set_gov_scripts( -// lua::Interpreter().invoke(operator_gov_script_file)); -// gen.finalize(); - -// StubNodeState node; -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// size_t proposal_id; -// size_t proposer_id = 1; -// size_t voter_id = 2; - -// const ccf::Script vote_for("return true"); -// const ccf::Script vote_against("return false"); -// { -// DOCTEST_INFO("Propose and vote for"); - -// const auto proposed_member = get_cert_data(4, kp); - -// Propose::In proposal; -// proposal.script = std::string(R"xxx( -// tables, member_info = ... -// return Calls:call("new_member", member_info) -// )xxx"); -// proposal.parameter["cert"] = proposed_member; -// proposal.parameter["keyshare"] = dummy_key_share; -// proposal.ballot = vote_for; - -// const auto propose = create_signed_request(proposal, "propose", kp); -// const auto r = parse_response_body(frontend_process( -// frontend, -// propose, -// tls::make_verifier(members[proposer_id])->der_cert_data())); - -// DOCTEST_CHECK(r.state == ProposalState::OPEN); - -// proposal_id = r.proposal_id; -// } - -// { -// DOCTEST_INFO("Operator votes, but without effect"); - -// const auto vote = -// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); -// const auto r = frontend_process(frontend, vote, operator_cert); - -// check_result_state(r, ProposalState::OPEN); -// } - -// { -// DOCTEST_INFO("Second member votes for proposal, which passes"); - -// const auto vote = -// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); -// const auto r = frontend_process(frontend, vote, members[voter_id]); - -// check_result_state(r, ProposalState::ACCEPTED); -// } - -// { -// DOCTEST_INFO("Validate vote tally"); - -// const auto readj = create_signed_request( -// read_params(proposal_id, Tables::PROPOSALS), "read", kp); - -// const auto proposal = -// get_proposal(frontend, proposal_id, members[proposer_id]); - -// const auto& votes = proposal.votes; -// DOCTEST_CHECK(votes.size() == 3); - -// const auto operator_vote = votes.find(operator_id); -// DOCTEST_CHECK(operator_vote != votes.end()); -// DOCTEST_CHECK(operator_vote->second == vote_for); - -// const auto proposer_vote = votes.find(proposer_id); -// DOCTEST_CHECK(proposer_vote != votes.end()); -// DOCTEST_CHECK(proposer_vote->second == vote_for); - -// const auto voter_vote = votes.find(voter_id); -// DOCTEST_CHECK(voter_vote != votes.end()); -// DOCTEST_CHECK(voter_vote->second == vote_for); -// } -// } - -// DOCTEST_TEST_CASE("Passing operator vote") -// { -// // Operator issues a proposal that only requires its own vote -// // and gets it through without member votes -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// auto new_kp = tls::make_key_pair(); -// auto new_ca = new_kp->self_sign("CN=new node"); -// NodeInfo ni; -// ni.cert = new_ca; -// gen.add_node(ni); - -// // Operating member, as set in operator_gov.lua -// const auto operator_cert = get_cert_data(0, kp); -// const auto operator_id = gen.add_member(operator_cert, {}); - -// // Non-operating members -// std::map members; -// for (size_t i = 1; i < 4; i++) -// { -// auto cert = get_cert_data(i, kp); -// members[gen.add_member(cert, {})] = cert; -// } - -// set_whitelists(gen); -// gen.set_gov_scripts( -// lua::Interpreter().invoke(operator_gov_script_file)); -// gen.finalize(); - -// StubNodeState node; -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// size_t proposal_id; - -// const ccf::Script vote_for("return true"); -// const ccf::Script vote_against("return false"); - -// auto node_id = 0; -// { -// DOCTEST_INFO("Check node exists with status pending"); -// auto read_values = -// create_request(read_params(node_id, Tables::NODES), "read"); -// const auto r = parse_response_body( -// frontend_process(frontend, read_values, operator_cert)); - -// DOCTEST_CHECK(r.status == NodeStatus::PENDING); -// } - -// { -// DOCTEST_INFO("Operator proposes and votes for node"); -// Script proposal(R"xxx( -// local tables, node_id = ... -// return Calls:call("trust_node", node_id) -// )xxx"); - -// const auto propose = create_signed_request( -// Propose::In{proposal, node_id, vote_for}, "propose", kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, operator_cert)); - -// DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); -// proposal_id = r.proposal_id; -// } - -// { -// DOCTEST_INFO("Validate vote tally"); - -// const auto readj = create_signed_request( -// read_params(proposal_id, Tables::PROPOSALS), "read", kp); - -// const auto proposal = get_proposal(frontend, proposal_id, operator_cert); - -// const auto& votes = proposal.votes; -// DOCTEST_CHECK(votes.size() == 1); - -// const auto proposer_vote = votes.find(operator_id); -// DOCTEST_CHECK(proposer_vote != votes.end()); -// DOCTEST_CHECK(proposer_vote->second == vote_for); -// } -// } - -// DOCTEST_TEST_CASE("Members passing an operator vote") -// { -// // Operator proposes a vote, but does not vote for it -// // A majority of members pass the vote -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// auto new_kp = tls::make_key_pair(); -// auto new_ca = new_kp->self_sign("CN=new node"); -// NodeInfo ni; -// ni.cert = new_ca; -// gen.add_node(ni); - -// // Operating member, as set in operator_gov.lua -// const auto operator_cert = get_cert_data(0, kp); -// const auto operator_id = gen.add_member(operator_cert, {}); - -// // Non-operating members -// std::map members; -// for (size_t i = 1; i < 4; i++) -// { -// auto cert = get_cert_data(i, kp); -// members[gen.add_member(cert, {})] = cert; -// } - -// set_whitelists(gen); -// gen.set_gov_scripts( -// lua::Interpreter().invoke(operator_gov_script_file)); -// gen.finalize(); - -// StubNodeState node; -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// size_t proposal_id; - -// const ccf::Script vote_for("return true"); -// const ccf::Script vote_against("return false"); - -// auto node_id = 0; -// { -// DOCTEST_INFO("Check node exists with status pending"); -// const auto read_values = -// create_request(read_params(node_id, Tables::NODES), "read"); -// const auto r = parse_response_body( -// frontend_process(frontend, read_values, operator_cert)); -// DOCTEST_CHECK(r.status == NodeStatus::PENDING); -// } - -// { -// DOCTEST_INFO("Operator proposes and votes against adding node"); -// Script proposal(R"xxx( -// local tables, node_id = ... -// return Calls:call("trust_node", node_id) -// )xxx"); - -// const auto propose = create_signed_request( -// Propose::In{proposal, node_id, vote_against}, "propose", kp); -// const auto r = parse_response_body( -// frontend_process(frontend, propose, operator_cert)); - -// DOCTEST_CHECK(r.state == ProposalState::OPEN); -// proposal_id = r.proposal_id; -// } - -// size_t first_voter_id = 1; -// size_t second_voter_id = 2; - -// { -// DOCTEST_INFO("First member votes for proposal"); - -// const auto vote = -// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); -// const auto r = frontend_process(frontend, vote, members[first_voter_id]); - -// check_result_state(r, ProposalState::OPEN); -// } - -// { -// DOCTEST_INFO("Second member votes for proposal"); - -// const auto vote = -// create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); -// const auto r = frontend_process(frontend, vote, -// members[second_voter_id]); - -// check_result_state(r, ProposalState::ACCEPTED); -// } - -// { -// DOCTEST_INFO("Validate vote tally"); - -// const auto readj = create_signed_request( -// read_params(proposal_id, Tables::PROPOSALS), "read", kp); - -// const auto proposal = get_proposal(frontend, proposal_id, operator_cert); - -// const auto& votes = proposal.votes; -// DOCTEST_CHECK(votes.size() == 3); - -// const auto proposer_vote = votes.find(operator_id); -// DOCTEST_CHECK(proposer_vote != votes.end()); -// DOCTEST_CHECK(proposer_vote->second == vote_against); - -// const auto first_vote = votes.find(first_voter_id); -// DOCTEST_CHECK(first_vote != votes.end()); -// DOCTEST_CHECK(first_vote->second == vote_for); - -// const auto second_vote = votes.find(second_voter_id); -// DOCTEST_CHECK(second_vote != votes.end()); -// DOCTEST_CHECK(second_vote->second == vote_for); -// } -// } - -// DOCTEST_TEST_CASE("User data") -// { -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// const auto member_id = gen.add_member(member_cert, {}); -// const auto user_id = gen.add_user(user_cert); -// set_whitelists(gen); -// gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); -// gen.finalize(); - -// StubNodeState node; -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// const auto read_user_info = -// create_request(read_params(user_id, Tables::USERS), "read"); - -// { -// DOCTEST_INFO("user data is initially empty"); -// const auto read_response = parse_response_body( -// frontend_process(frontend, read_user_info, member_cert)); -// DOCTEST_CHECK(read_response.user_data.is_null()); -// } - -// { -// auto user_data_object = nlohmann::json::object(); -// user_data_object["name"] = "bob"; -// user_data_object["permissions"] = {"read", "delete"}; - -// DOCTEST_INFO("user data can be set to an object"); -// Propose::In proposal; -// proposal.script = fmt::format( -// R"xxx( -// proposed_user_data = {{ -// name = "bob", -// permissions = {{"read", "delete"}} -// }} -// return Calls:call("set_user_data", {{user_id = {}, user_data = -// proposed_user_data}}) -// )xxx", -// user_id); -// const auto proposal_serialized = -// create_signed_request(proposal, "propose", kp); -// const auto propose_response = parse_response_body( -// frontend_process(frontend, proposal_serialized, member_cert)); -// DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); - -// DOCTEST_INFO("user data object can be read"); -// const auto read_response = parse_response_body( -// frontend_process(frontend, read_user_info, member_cert)); -// DOCTEST_CHECK(read_response.user_data == user_data_object); -// } - -// { -// const auto user_data_string = "ADMINISTRATOR"; - -// DOCTEST_INFO("user data can be overwritten"); -// Propose::In proposal; -// proposal.script = std::string(R"xxx( -// local tables, param = ... -// return Calls:call("set_user_data", {user_id = param.id, user_data = -// param.data}) -// )xxx"); -// proposal.parameter["id"] = user_id; -// proposal.parameter["data"] = user_data_string; -// const auto proposal_serialized = -// create_signed_request(proposal, "propose", kp); -// const auto propose_response = parse_response_body( -// frontend_process(frontend, proposal_serialized, member_cert)); -// DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); - -// DOCTEST_INFO("user data object can be read"); -// const auto response = parse_response_body( -// frontend_process(frontend, read_user_info, member_cert)); -// DOCTEST_CHECK(response.user_data == user_data_string); -// } -// } +std::vector gen_public_encryption_key() +{ + auto private_encryption_key = + tls::create_entropy()->random(crypto::BoxKey::KEY_SIZE); + return tls::PublicX25519::write( + crypto::BoxKey::public_from_private(private_encryption_key)) + .raw(); +} + +auto init_frontend( + NetworkTables& network, + GenesisGenerator& gen, + StubNodeState& node, + ShareManager& share_manager, + const int n_members, + std::vector>& member_certs) +{ + // create members + for (uint8_t i = 0; i < n_members; i++) + { + member_certs.push_back(get_cert_data(i, kp)); + gen.add_member(member_certs.back(), {}); + } + + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); + gen.finalize(); + + return MemberRpcFrontend(network, node, share_manager); +} + +DOCTEST_TEST_CASE("Member query/read") +{ + // initialize the network state + NetworkState network; + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + const auto member_id = gen.add_member(member_cert, {}); + gen.finalize(); + + const enclave::SessionContext member_session( + enclave::InvalidSessionId, member_cert); + + // put value to read + constexpr auto key = 123; + constexpr auto value = 456; + Store::Tx tx; + tx.get_view(network.values)->put(key, value); + DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + + static constexpr auto query = R"xxx( + local tables = ... + return tables["ccf.values"]:get(123) + )xxx"; + + DOCTEST_SUBCASE("Query: bytecode/script allowed access") + { + // set member ACL so that the VALUES table is accessible + Store::Tx tx; + tx.get_view(network.whitelists) + ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); + DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + + bool compile = true; + do + { + const auto req = create_request(query_params(query, compile), "query"); + const auto r = frontend_process(frontend, req, member_cert); + const auto result = parse_response_body(r); + DOCTEST_CHECK(result == value); + compile = !compile; + } while (!compile); + } + + DOCTEST_SUBCASE("Query: table not in ACL") + { + // set member ACL so that no table is accessible + Store::Tx tx; + tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); + DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + + auto req = create_request(query_params(query, true), "query"); + const auto response = frontend_process(frontend, req, member_cert); + + check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); + } + + DOCTEST_SUBCASE("Read: allowed access, key exists") + { + Store::Tx tx; + tx.get_view(network.whitelists) + ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); + DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + + auto read_call = + create_request(read_params(key, Tables::VALUES), "read"); + const auto r = frontend_process(frontend, read_call, member_cert); + const auto result = parse_response_body(r); + DOCTEST_CHECK(result == value); + } + + DOCTEST_SUBCASE("Read: allowed access, key doesn't exist") + { + constexpr auto wrong_key = 321; + Store::Tx tx; + tx.get_view(network.whitelists) + ->put(WlIds::MEMBER_CAN_READ, {Tables::VALUES}); + DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + + auto read_call = + create_request(read_params(wrong_key, Tables::VALUES), "read"); + const auto response = frontend_process(frontend, read_call, member_cert); + + check_error(response, HTTP_STATUS_BAD_REQUEST); + } + + DOCTEST_SUBCASE("Read: access not allowed") + { + Store::Tx tx; + tx.get_view(network.whitelists)->put(WlIds::MEMBER_CAN_READ, {}); + DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + + auto read_call = + create_request(read_params(key, Tables::VALUES), "read"); + const auto response = frontend_process(frontend, read_call, member_cert); + + check_error(response, HTTP_STATUS_INTERNAL_SERVER_ERROR); + } +} + +DOCTEST_TEST_CASE("Proposer ballot") +{ + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + + const auto proposer_cert = get_cert_data(0, kp); + const auto proposer_id = gen.add_member(proposer_cert, {}); + const auto voter_cert = get_cert_data(1, kp); + const auto voter_id = gen.add_member(voter_cert, {}); + + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); + gen.finalize(); + + ShareManager share_manager(network); + StubNodeState node(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + size_t proposal_id; + + const ccf::Script vote_for("return true"); + const ccf::Script vote_against("return false"); + { + DOCTEST_INFO("Propose, initially voting against"); + + const auto proposed_member = get_cert_data(2, kp); + + Propose::In proposal; + proposal.script = std::string(R"xxx( + tables, member_info = ... + return Calls:call("new_member", member_info) + )xxx"); + proposal.parameter["cert"] = proposed_member; + proposal.parameter["keyshare"] = dummy_key_share; + proposal.ballot = vote_against; + const auto propose = create_signed_request(proposal, "propose", kp); + const auto r = frontend_process(frontend, propose, proposer_cert); + + // the proposal should be accepted, but not succeed immediately + const auto result = parse_response_body(r); + DOCTEST_CHECK(result.state == ProposalState::OPEN); + + proposal_id = result.proposal_id; + } + + { + DOCTEST_INFO("Second member votes for proposal"); + + const auto vote = + create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); + const auto r = frontend_process(frontend, vote, voter_cert); + + // The vote should not yet succeed + check_result_state(r, ProposalState::OPEN); + } + + { + DOCTEST_INFO("Read current votes"); + + const auto proposal_result = + get_proposal(frontend, proposal_id, proposer_cert); + + const auto& votes = proposal_result.votes; + DOCTEST_CHECK(votes.size() == 2); + + const auto proposer_vote = votes.find(proposer_id); + DOCTEST_CHECK(proposer_vote != votes.end()); + DOCTEST_CHECK(proposer_vote->second == vote_against); + + const auto voter_vote = votes.find(voter_id); + DOCTEST_CHECK(voter_vote != votes.end()); + DOCTEST_CHECK(voter_vote->second == vote_for); + } + + { + DOCTEST_INFO("Proposer votes for"); + + const auto vote = + create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); + const auto r = frontend_process(frontend, vote, proposer_cert); + + // The vote should now succeed + check_result_state(r, ProposalState::ACCEPTED); + } +} + +struct NewMember +{ + MemberId id; + tls::KeyPairPtr kp = tls::make_key_pair(); + Cert cert; +}; + +DOCTEST_TEST_CASE("Add new members until there are 7 then reject") +{ + logger::config::level() = logger::INFO; + + constexpr auto initial_members = 3; + constexpr auto n_new_members = 7; + constexpr auto max_members = 8; + NetworkState network; + network.ledger_secrets = std::make_shared(); + network.ledger_secrets->init(); + network.encryption_key = std::make_unique( + tls::create_entropy()->random(crypto::BoxKey::KEY_SIZE)); + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + // add three initial active members + // the proposer + auto proposer_id = gen.add_member(member_cert, gen_public_encryption_key()); + + // the voters + const auto voter_a_cert = get_cert_data(1, kp); + auto voter_a = gen.add_member(voter_a_cert, gen_public_encryption_key()); + const auto voter_b_cert = get_cert_data(2, kp); + auto voter_b = gen.add_member(voter_b_cert, gen_public_encryption_key()); + + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); + gen.set_recovery_threshold(1); + gen.open_service(); + gen.finalize(); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + vector new_members(n_new_members); + + auto i = 0ul; + for (auto& new_member : new_members) + { + const auto proposal_id = i; + new_member.id = initial_members + i++; + + // new member certificate + auto cert_pem = + new_member.kp->self_sign(fmt::format("CN=new member{}", new_member.id)); + auto keyshare = dummy_key_share; + auto v = tls::make_verifier(cert_pem); + const auto _cert = v->raw(); + new_member.cert = {_cert->raw.p, _cert->raw.p + _cert->raw.len}; + + // check new_member id does not work before member is added + const auto read_next_req = create_request( + read_params(ValueIds::NEXT_MEMBER_ID, Tables::VALUES), "read"); + const auto r = frontend_process(frontend, read_next_req, new_member.cert); + check_error(r, HTTP_STATUS_FORBIDDEN); + + // propose new member, as proposer + Propose::In proposal; + proposal.script = std::string(R"xxx( + tables, member_info = ... + return Calls:call("new_member", member_info) + )xxx"); + proposal.parameter["cert"] = cert_pem; + proposal.parameter["keyshare"] = gen_public_encryption_key(); + + const auto propose = create_signed_request(proposal, "propose", kp); + + { + const auto r = frontend_process(frontend, propose, member_cert); + const auto result = parse_response_body(r); + + // the proposal should be accepted, but not succeed immediately + DOCTEST_CHECK(result.proposal_id == proposal_id); + DOCTEST_CHECK(result.state == ProposalState::OPEN); + } + + // read initial proposal, as second member + const Proposal initial_read = + get_proposal(frontend, proposal_id, voter_a_cert); + DOCTEST_CHECK(initial_read.proposer == proposer_id); + DOCTEST_CHECK(initial_read.script == proposal.script); + DOCTEST_CHECK(initial_read.parameter == proposal.parameter); + + // vote as second member + Script vote_ballot(fmt::format( + R"xxx( + local tables, calls = ... + local n = 0 + tables["ccf.members"]:foreach( function(k, v) n = n + 1 end ) + if n < {} then + return true + else + return false + end + )xxx", + max_members)); + + const auto vote = + create_signed_request(Vote{proposal_id, vote_ballot}, "vote", kp); + + { + const auto r = frontend_process(frontend, vote, voter_a_cert); + const auto result = parse_response_body(r); + + if (new_member.id < max_members) + { + // vote should succeed + DOCTEST_CHECK(result.state == ProposalState::ACCEPTED); + // check that member with the new new_member cert can make RPCs now + DOCTEST_CHECK( + parse_response_body(frontend_process( + frontend, read_next_req, new_member.cert)) == new_member.id + 1); + + // successful proposals are removed from the kv, so we can't confirm + // their final state + } + else + { + // vote should not succeed + DOCTEST_CHECK(result.state == ProposalState::OPEN); + // check that member with the new new_member cert can make RPCs now + check_error( + frontend_process(frontend, read_next_req, new_member.cert), + HTTP_STATUS_FORBIDDEN); + + // re-read proposal, as second member + const Proposal final_read = + get_proposal(frontend, proposal_id, voter_a_cert); + DOCTEST_CHECK(final_read.proposer == proposer_id); + DOCTEST_CHECK(final_read.script == proposal.script); + DOCTEST_CHECK(final_read.parameter == proposal.parameter); + + const auto my_vote = final_read.votes.find(voter_a); + DOCTEST_CHECK(my_vote != final_read.votes.end()); + DOCTEST_CHECK(my_vote->second == vote_ballot); + } + } + } + + DOCTEST_SUBCASE("ACK from newly added members") + { + // iterate over all new_members, except for the last one + for (auto new_member = new_members.cbegin(); new_member != + new_members.cend() - (initial_members + n_new_members - max_members); + new_member++) + { + // (1) read ack entry + const auto read_state_digest_req = create_request( + read_params(new_member->id, Tables::MEMBER_ACKS), "read"); + const auto ack0 = parse_response_body( + frontend_process(frontend, read_state_digest_req, new_member->cert)); + DOCTEST_REQUIRE(std::all_of( + ack0.state_digest.begin(), ack0.state_digest.end(), [](uint8_t i) { + return i == 0; + })); + + { + // make sure that there is a signature in the signatures table since + // ack's depend on that + Store::Tx tx; + auto sig_view = tx.get_view(network.signatures); + Signature sig_value; + sig_view->put(0, sig_value); + DOCTEST_REQUIRE(tx.commit() == kv::CommitSuccess::OK); + } + + // (2) ask for a fresher digest of state + const auto freshen_state_digest_req = + create_request(nullptr, "updateAckStateDigest"); + const auto freshen_state_digest = parse_response_body( + frontend_process(frontend, freshen_state_digest_req, new_member->cert)); + DOCTEST_CHECK(freshen_state_digest.state_digest != ack0.state_digest); + + // (3) read ack entry again and check that the state digest has changed + const auto ack1 = parse_response_body( + frontend_process(frontend, read_state_digest_req, new_member->cert)); + DOCTEST_CHECK(ack0.state_digest != ack1.state_digest); + DOCTEST_CHECK(freshen_state_digest.state_digest == ack1.state_digest); + + // (4) sign stale state and send it + StateDigest params; + params.state_digest = ack0.state_digest; + const auto send_stale_sig_req = + create_signed_request(params, "ack", new_member->kp); + check_error( + frontend_process(frontend, send_stale_sig_req, new_member->cert), + HTTP_STATUS_BAD_REQUEST); + + // (5) sign new state digest and send it + params.state_digest = ack1.state_digest; + const auto send_good_sig_req = + create_signed_request(params, "ack", new_member->kp); + const auto good_response = + frontend_process(frontend, send_good_sig_req, new_member->cert); + DOCTEST_CHECK(good_response.status == HTTP_STATUS_OK); + DOCTEST_CHECK(parse_response_body(good_response)); + + // (6) read own member status + const auto read_status_req = + create_request(read_params(new_member->id, Tables::MEMBERS), "read"); + const auto mi = parse_response_body( + frontend_process(frontend, read_status_req, new_member->cert)); + DOCTEST_CHECK(mi.status == MemberStatus::ACTIVE); + } + } +} + +DOCTEST_TEST_CASE("Accept node") +{ + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + auto new_kp = tls::make_key_pair(); + + const auto member_0_cert = get_cert_data(0, new_kp); + const auto member_1_cert = get_cert_data(1, kp); + const auto member_0 = gen.add_member(member_0_cert, {}); + const auto member_1 = gen.add_member(member_1_cert, {}); + + // node to be tested + // new node certificate + auto new_ca = new_kp->self_sign("CN=new node"); + NodeInfo ni; + ni.cert = new_ca; + gen.add_node(ni); + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); + gen.finalize(); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + auto node_id = 0; + + // check node exists with status pending + { + auto read_values = + create_request(read_params(node_id, Tables::NODES), "read"); + const auto r = parse_response_body( + frontend_process(frontend, read_values, member_0_cert)); + + DOCTEST_CHECK(r.status == NodeStatus::PENDING); + } + + // m0 proposes adding new node + { + Script proposal(R"xxx( + local tables, node_id = ... + return Calls:call("trust_node", node_id) + )xxx"); + const auto propose = + create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, member_0_cert)); + + DOCTEST_CHECK(r.state == ProposalState::OPEN); + DOCTEST_CHECK(r.proposal_id == 0); + } + + // m1 votes for accepting a single new node + { + Script vote_ballot(R"xxx( + local tables, calls = ... + return #calls == 1 and calls[1].func == "trust_node" + )xxx"); + const auto vote = create_signed_request(Vote{0, vote_ballot}, "vote", kp); + + check_result_state( + frontend_process(frontend, vote, member_1_cert), ProposalState::ACCEPTED); + } + + // check node exists with status pending + { + const auto read_values = + create_request(read_params(node_id, Tables::NODES), "read"); + const auto r = parse_response_body( + frontend_process(frontend, read_values, member_0_cert)); + DOCTEST_CHECK(r.status == NodeStatus::TRUSTED); + } + + // m0 proposes retire node + { + Script proposal(R"xxx( + local tables, node_id = ... + return Calls:call("retire_node", node_id) + )xxx"); + const auto propose = + create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, member_0_cert)); + + DOCTEST_CHECK(r.state == ProposalState::OPEN); + DOCTEST_CHECK(r.proposal_id == 1); + } + + // m1 votes for retiring node + { + const Script vote_ballot("return true"); + const auto vote = create_signed_request(Vote{1, vote_ballot}, "vote", kp); + check_result_state( + frontend_process(frontend, vote, member_1_cert), ProposalState::ACCEPTED); + } + + // check that node exists with status retired + { + auto read_values = + create_request(read_params(node_id, Tables::NODES), "read"); + const auto r = parse_response_body( + frontend_process(frontend, read_values, member_0_cert)); + DOCTEST_CHECK(r.status == NodeStatus::RETIRED); + } + + // check that retired node cannot be trusted + { + Script proposal(R"xxx( + local tables, node_id = ... + return Calls:call("trust_node", node_id) + )xxx"); + const auto propose = + create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, member_0_cert)); + + const Script vote_ballot("return true"); + const auto vote = + create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); + check_result_state( + frontend_process(frontend, vote, member_1_cert), ProposalState::FAILED); + } + + // check that retired node cannot be retired again + { + Script proposal(R"xxx( + local tables, node_id = ... + return Calls:call("retire_node", node_id) + )xxx"); + const auto propose = + create_signed_request(Propose::In{proposal, node_id}, "propose", new_kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, member_0_cert)); + + const Script vote_ballot("return true"); + const auto vote = + create_signed_request(Vote{r.proposal_id, vote_ballot}, "vote", kp); + check_result_state( + frontend_process(frontend, vote, member_1_cert), ProposalState::FAILED); + } +} + +ProposalInfo test_raw_writes( + NetworkTables& network, + GenesisGenerator& gen, + StubNodeState& node, + ShareManager& share_manager, + Propose::In proposal, + const int n_members = 1, + const int pro_votes = 1, + bool explicit_proposer_vote = false) +{ + std::vector> member_certs; + auto frontend = + init_frontend(network, gen, node, share_manager, n_members, member_certs); + frontend.open(); + + // check values before + { + Store::Tx tx; + auto next_member_id_r = + tx.get_view(network.values)->get(ValueIds::NEXT_MEMBER_ID); + DOCTEST_CHECK(next_member_id_r); + DOCTEST_CHECK(*next_member_id_r == n_members); + } + + // propose + const auto proposal_id = 0ul; + { + const uint8_t proposer_id = 0; + const auto propose = create_signed_request(proposal, "propose", kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, member_certs[0])); + + const auto expected_state = + (n_members == 1) ? ProposalState::ACCEPTED : ProposalState::OPEN; + DOCTEST_CHECK(r.state == expected_state); + DOCTEST_CHECK(r.proposal_id == proposal_id); + if (r.state == ProposalState::ACCEPTED) + return r; + } + + // con votes + for (int i = n_members - 1; i >= pro_votes; i--) + { + const Script vote("return false"); + const auto vote_serialized = + create_signed_request(Vote{proposal_id, vote}, "vote", kp); + + check_result_state( + frontend_process(frontend, vote_serialized, member_certs[i]), + ProposalState::OPEN); + } + + // pro votes (proposer also votes) + ProposalInfo info = {}; + for (uint8_t i = explicit_proposer_vote ? 0 : 1; i < pro_votes; i++) + { + const Script vote("return true"); + const auto vote_serialized = + create_signed_request(Vote{proposal_id, vote}, "vote", kp); + if (info.state == ProposalState::OPEN) + { + info = parse_response_body( + frontend_process(frontend, vote_serialized, member_certs[i])); + } + else + { + // proposal has been accepted - additional votes return an error + check_error( + frontend_process(frontend, vote_serialized, member_certs[i]), + HTTP_STATUS_BAD_REQUEST); + } + } + return info; +} + +DOCTEST_TEST_CASE("Propose raw writes") +{ + logger::config::level() = logger::INFO; + DOCTEST_SUBCASE("insensitive tables") + { + const auto n_members = 3; + for (int pro_votes = 0; pro_votes <= n_members; pro_votes++) + { + const bool should_succeed = pro_votes > n_members / 2; + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + nlohmann::json recovery_threshold = 4; + + Store::Tx tx_before; + auto configuration = tx_before.get_view(network.config)->get(0); + DOCTEST_REQUIRE_FALSE(configuration.has_value()); + + const auto expected_state = + should_succeed ? ProposalState::ACCEPTED : ProposalState::OPEN; + const auto proposal_info = test_raw_writes( + network, + gen, + node, + share_manager, + {R"xxx( + local tables, recovery_threshold = ... + local p = Puts:new() + p:put("ccf.config", 0, {recovery_threshold = recovery_threshold}) + return Calls:call("raw_puts", p) + )xxx"s, + 4}, + n_members, + pro_votes); + DOCTEST_CHECK(proposal_info.state == expected_state); + if (!should_succeed) + continue; + + // check results + Store::Tx tx_after; + configuration = tx_after.get_view(network.config)->get(0); + DOCTEST_CHECK(configuration.has_value()); + DOCTEST_CHECK(configuration->recovery_threshold == recovery_threshold); + } + } + + DOCTEST_SUBCASE("sensitive tables") + { + // propose changes to sensitive tables; changes must only be accepted + // unanimously create new network for each case + const auto sensitive_tables = {Tables::WHITELISTS, Tables::GOV_SCRIPTS}; + const auto n_members = 3; + // let proposer vote/not vote + for (const auto proposer_vote : {true, false}) + { + for (int pro_votes = 0; pro_votes < n_members; pro_votes++) + { + for (const auto& sensitive_table : sensitive_tables) + { + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + + const auto sensitive_put = + "return Calls:call('raw_puts', Puts:put('"s + sensitive_table + + "', 9, {'aaa'}))"s; + const auto expected_state = (n_members == pro_votes) ? + ProposalState::ACCEPTED : + ProposalState::OPEN; + const auto proposal_info = test_raw_writes( + network, + gen, + node, + share_manager, + {sensitive_put}, + n_members, + pro_votes, + proposer_vote); + DOCTEST_CHECK(proposal_info.state == expected_state); + } + } + } + } +} + +DOCTEST_TEST_CASE("Remove proposal") +{ + NewMember caller; + auto cert = caller.kp->self_sign("CN=new member"); + auto v = tls::make_verifier(cert); + caller.cert = v->der_cert_data(); + + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + + ShareManager share_manager(network); + StubNodeState node(share_manager); + gen.add_member(member_cert, {}); + gen.add_member(cert, {}); + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); + gen.finalize(); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + auto proposal_id = 0; + auto wrong_proposal_id = 1; + ccf::Script proposal_script(R"xxx( + local tables, param = ... + return {} + )xxx"); + + // check that the proposal doesn't exist + { + Store::Tx tx; + auto proposal = tx.get_view(network.proposals)->get(proposal_id); + DOCTEST_CHECK(!proposal); + } + + { + const auto propose = + create_signed_request(Propose::In{proposal_script, 0}, "propose", kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, member_cert)); + + DOCTEST_CHECK(r.proposal_id == proposal_id); + DOCTEST_CHECK(r.state == ProposalState::OPEN); + } + + // check that the proposal is there + { + Store::Tx tx; + auto proposal = tx.get_view(network.proposals)->get(proposal_id); + DOCTEST_CHECK(proposal); + DOCTEST_CHECK(proposal->state == ProposalState::OPEN); + DOCTEST_CHECK( + proposal->script.text.value() == proposal_script.text.value()); + } + + DOCTEST_SUBCASE("Attempt withdraw proposal with non existing id") + { + json param; + param["id"] = wrong_proposal_id; + const auto withdraw = create_signed_request(param, "withdraw", kp); + + check_error( + frontend_process(frontend, withdraw, member_cert), + HTTP_STATUS_BAD_REQUEST); + } + + DOCTEST_SUBCASE("Attempt withdraw proposal that you didn't propose") + { + json param; + param["id"] = proposal_id; + const auto withdraw = create_signed_request(param, "withdraw", caller.kp); + + check_error( + frontend_process(frontend, withdraw, cert), HTTP_STATUS_FORBIDDEN); + } + + DOCTEST_SUBCASE("Successfully withdraw proposal") + { + json param; + param["id"] = proposal_id; + const auto withdraw = create_signed_request(param, "withdraw", kp); + + check_result_state( + frontend_process(frontend, withdraw, member_cert), + ProposalState::WITHDRAWN); + + // check that the proposal is now withdrawn + { + Store::Tx tx; + auto proposal = tx.get_view(network.proposals)->get(proposal_id); + DOCTEST_CHECK(proposal.has_value()); + DOCTEST_CHECK(proposal->state == ProposalState::WITHDRAWN); + } + } +} + +DOCTEST_TEST_CASE("Complete proposal after initial rejection") +{ + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + std::vector> member_certs; + auto frontend = + init_frontend(network, gen, node, share_manager, 3, member_certs); + frontend.open(); + + { + DOCTEST_INFO("Propose"); + const auto proposal = + "return Calls:call('raw_puts', Puts:put('ccf.values', 999, 999))"s; + const auto propose = + create_signed_request(Propose::In{proposal}, "propose", kp); + + Store::Tx tx; + const auto r = parse_response_body( + frontend_process(frontend, propose, member_certs[0])); + DOCTEST_CHECK(r.state == ProposalState::OPEN); + } + + { + DOCTEST_INFO("Vote that rejects initially"); + const Script vote(R"xxx( + local tables = ... + return tables["ccf.values"]:get(123) == 123 + )xxx"); + const auto vote_serialized = + create_signed_request(Vote{0, vote}, "vote", kp); + + check_result_state( + frontend_process(frontend, vote_serialized, member_certs[1]), + ProposalState::OPEN); + } + + { + DOCTEST_INFO("Try to complete"); + const auto complete = + create_signed_request(ProposalAction{0}, "complete", kp); + + check_result_state( + frontend_process(frontend, complete, member_certs[1]), + ProposalState::OPEN); + } + + { + DOCTEST_INFO("Put value that makes vote agree"); + Store::Tx tx; + tx.get_view(network.values)->put(123, 123); + DOCTEST_CHECK(tx.commit() == kv::CommitSuccess::OK); + } + + { + DOCTEST_INFO("Try again to complete"); + const auto complete = + create_signed_request(ProposalAction{0}, "complete", kp); + + check_result_state( + frontend_process(frontend, complete, member_certs[1]), + ProposalState::ACCEPTED); + } +} + +DOCTEST_TEST_CASE("Vetoed proposal gets rejected") +{ + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + const auto voter_a_cert = get_cert_data(1, kp); + auto voter_a = gen.add_member(voter_a_cert, {}); + const auto voter_b_cert = get_cert_data(2, kp); + auto voter_b = gen.add_member(voter_b_cert, {}); + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_veto_script_file)); + gen.finalize(); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + Script proposal(R"xxx( + tables, user_cert = ... + return Calls:call("new_user", user_cert) + )xxx"); + + const vector user_cert = kp->self_sign("CN=new user"); + const auto propose = + create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); + + const auto r = parse_response_body( + frontend_process(frontend, propose, voter_a_cert)); + DOCTEST_CHECK(r.state == ProposalState::OPEN); + DOCTEST_CHECK(r.proposal_id == 0); + + const ccf::Script vote_against("return false"); + { + DOCTEST_INFO("Member vetoes proposal"); + + const auto vote = create_signed_request(Vote{0, vote_against}, "vote", kp); + const auto r = frontend_process(frontend, vote, voter_b_cert); + + check_result_state(r, ProposalState::REJECTED); + } + + { + DOCTEST_INFO("Check proposal was rejected"); + + const auto proposal = get_proposal(frontend, 0, voter_a_cert); + + DOCTEST_CHECK(proposal.state == ProposalState::REJECTED); + } +} + +DOCTEST_TEST_CASE("Add user via proposed call") +{ + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + ShareManager share_manager(network); + StubNodeState node(share_manager); + const auto member_cert = get_cert_data(0, kp); + gen.add_member(member_cert, {}); + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); + gen.finalize(); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + Script proposal(R"xxx( + tables, user_cert = ... + return Calls:call("new_user", user_cert) + )xxx"); + + const vector user_cert = kp->self_sign("CN=new user"); + const auto propose = + create_signed_request(Propose::In{proposal, user_cert}, "propose", kp); + + const auto r = parse_response_body( + frontend_process(frontend, propose, member_cert)); + DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); + DOCTEST_CHECK(r.proposal_id == 0); + + Store::Tx tx1; + const auto uid = tx1.get_view(network.values)->get(ValueIds::NEXT_USER_ID); + DOCTEST_CHECK(uid); + DOCTEST_CHECK(*uid == 1); + const auto uid1 = tx1.get_view(network.user_certs) + ->get(tls::make_verifier(user_cert)->der_cert_data()); + DOCTEST_CHECK(uid1); + DOCTEST_CHECK(*uid1 == 0); +} + +DOCTEST_TEST_CASE("Passing members ballot with operator") +{ + // Members pass a ballot with a constitution that includes an operator + // Operator votes, but is _not_ taken into consideration + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + + // Operating member, as set in operator_gov.lua + const auto operator_cert = get_cert_data(0, kp); + const auto operator_id = gen.add_member(operator_cert, {}); + + // Non-operating members + std::map members; + for (size_t i = 1; i < 4; i++) + { + auto cert = get_cert_data(i, kp); + members[gen.add_member(cert, {})] = cert; + } + + set_whitelists(gen); + gen.set_gov_scripts( + lua::Interpreter().invoke(operator_gov_script_file)); + gen.finalize(); + + ShareManager share_manager(network); + StubNodeState node(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + size_t proposal_id; + size_t proposer_id = 1; + size_t voter_id = 2; + + const ccf::Script vote_for("return true"); + const ccf::Script vote_against("return false"); + { + DOCTEST_INFO("Propose and vote for"); + + const auto proposed_member = get_cert_data(4, kp); + + Propose::In proposal; + proposal.script = std::string(R"xxx( + tables, member_info = ... + return Calls:call("new_member", member_info) + )xxx"); + proposal.parameter["cert"] = proposed_member; + proposal.parameter["keyshare"] = dummy_key_share; + proposal.ballot = vote_for; + + const auto propose = create_signed_request(proposal, "propose", kp); + const auto r = parse_response_body(frontend_process( + frontend, + propose, + tls::make_verifier(members[proposer_id])->der_cert_data())); + + DOCTEST_CHECK(r.state == ProposalState::OPEN); + + proposal_id = r.proposal_id; + } + + { + DOCTEST_INFO("Operator votes, but without effect"); + + const auto vote = + create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); + const auto r = frontend_process(frontend, vote, operator_cert); + + check_result_state(r, ProposalState::OPEN); + } + + { + DOCTEST_INFO("Second member votes for proposal, which passes"); + + const auto vote = + create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); + const auto r = frontend_process(frontend, vote, members[voter_id]); + + check_result_state(r, ProposalState::ACCEPTED); + } + + { + DOCTEST_INFO("Validate vote tally"); + + const auto readj = create_signed_request( + read_params(proposal_id, Tables::PROPOSALS), "read", kp); + + const auto proposal = + get_proposal(frontend, proposal_id, members[proposer_id]); + + const auto& votes = proposal.votes; + DOCTEST_CHECK(votes.size() == 3); + + const auto operator_vote = votes.find(operator_id); + DOCTEST_CHECK(operator_vote != votes.end()); + DOCTEST_CHECK(operator_vote->second == vote_for); + + const auto proposer_vote = votes.find(proposer_id); + DOCTEST_CHECK(proposer_vote != votes.end()); + DOCTEST_CHECK(proposer_vote->second == vote_for); + + const auto voter_vote = votes.find(voter_id); + DOCTEST_CHECK(voter_vote != votes.end()); + DOCTEST_CHECK(voter_vote->second == vote_for); + } +} + +DOCTEST_TEST_CASE("Passing operator vote") +{ + // Operator issues a proposal that only requires its own vote + // and gets it through without member votes + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + auto new_kp = tls::make_key_pair(); + auto new_ca = new_kp->self_sign("CN=new node"); + NodeInfo ni; + ni.cert = new_ca; + gen.add_node(ni); + + // Operating member, as set in operator_gov.lua + const auto operator_cert = get_cert_data(0, kp); + const auto operator_id = gen.add_member(operator_cert, {}); + + // Non-operating members + std::map members; + for (size_t i = 1; i < 4; i++) + { + auto cert = get_cert_data(i, kp); + members[gen.add_member(cert, {})] = cert; + } + + set_whitelists(gen); + gen.set_gov_scripts( + lua::Interpreter().invoke(operator_gov_script_file)); + gen.finalize(); + + ShareManager share_manager(network); + StubNodeState node(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + size_t proposal_id; + + const ccf::Script vote_for("return true"); + const ccf::Script vote_against("return false"); + + auto node_id = 0; + { + DOCTEST_INFO("Check node exists with status pending"); + auto read_values = + create_request(read_params(node_id, Tables::NODES), "read"); + const auto r = parse_response_body( + frontend_process(frontend, read_values, operator_cert)); + + DOCTEST_CHECK(r.status == NodeStatus::PENDING); + } + + { + DOCTEST_INFO("Operator proposes and votes for node"); + Script proposal(R"xxx( + local tables, node_id = ... + return Calls:call("trust_node", node_id) + )xxx"); + + const auto propose = create_signed_request( + Propose::In{proposal, node_id, vote_for}, "propose", kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, operator_cert)); + + DOCTEST_CHECK(r.state == ProposalState::ACCEPTED); + proposal_id = r.proposal_id; + } + + { + DOCTEST_INFO("Validate vote tally"); + + const auto readj = create_signed_request( + read_params(proposal_id, Tables::PROPOSALS), "read", kp); + + const auto proposal = get_proposal(frontend, proposal_id, operator_cert); + + const auto& votes = proposal.votes; + DOCTEST_CHECK(votes.size() == 1); + + const auto proposer_vote = votes.find(operator_id); + DOCTEST_CHECK(proposer_vote != votes.end()); + DOCTEST_CHECK(proposer_vote->second == vote_for); + } +} + +DOCTEST_TEST_CASE("Members passing an operator vote") +{ + // Operator proposes a vote, but does not vote for it + // A majority of members pass the vote + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + auto new_kp = tls::make_key_pair(); + auto new_ca = new_kp->self_sign("CN=new node"); + NodeInfo ni; + ni.cert = new_ca; + gen.add_node(ni); + + // Operating member, as set in operator_gov.lua + const auto operator_cert = get_cert_data(0, kp); + const auto operator_id = gen.add_member(operator_cert, {}); + + // Non-operating members + std::map members; + for (size_t i = 1; i < 4; i++) + { + auto cert = get_cert_data(i, kp); + members[gen.add_member(cert, {})] = cert; + } + + set_whitelists(gen); + gen.set_gov_scripts( + lua::Interpreter().invoke(operator_gov_script_file)); + gen.finalize(); + + ShareManager share_manager(network); + StubNodeState node(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + size_t proposal_id; + + const ccf::Script vote_for("return true"); + const ccf::Script vote_against("return false"); + + auto node_id = 0; + { + DOCTEST_INFO("Check node exists with status pending"); + const auto read_values = + create_request(read_params(node_id, Tables::NODES), "read"); + const auto r = parse_response_body( + frontend_process(frontend, read_values, operator_cert)); + DOCTEST_CHECK(r.status == NodeStatus::PENDING); + } + + { + DOCTEST_INFO("Operator proposes and votes against adding node"); + Script proposal(R"xxx( + local tables, node_id = ... + return Calls:call("trust_node", node_id) + )xxx"); + + const auto propose = create_signed_request( + Propose::In{proposal, node_id, vote_against}, "propose", kp); + const auto r = parse_response_body( + frontend_process(frontend, propose, operator_cert)); + + DOCTEST_CHECK(r.state == ProposalState::OPEN); + proposal_id = r.proposal_id; + } + + size_t first_voter_id = 1; + size_t second_voter_id = 2; + + { + DOCTEST_INFO("First member votes for proposal"); + + const auto vote = + create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); + const auto r = frontend_process(frontend, vote, members[first_voter_id]); + + check_result_state(r, ProposalState::OPEN); + } + + { + DOCTEST_INFO("Second member votes for proposal"); + + const auto vote = + create_signed_request(Vote{proposal_id, vote_for}, "vote", kp); + const auto r = frontend_process(frontend, vote, members[second_voter_id]); + + check_result_state(r, ProposalState::ACCEPTED); + } + + { + DOCTEST_INFO("Validate vote tally"); + + const auto readj = create_signed_request( + read_params(proposal_id, Tables::PROPOSALS), "read", kp); + + const auto proposal = get_proposal(frontend, proposal_id, operator_cert); + + const auto& votes = proposal.votes; + DOCTEST_CHECK(votes.size() == 3); + + const auto proposer_vote = votes.find(operator_id); + DOCTEST_CHECK(proposer_vote != votes.end()); + DOCTEST_CHECK(proposer_vote->second == vote_against); + + const auto first_vote = votes.find(first_voter_id); + DOCTEST_CHECK(first_vote != votes.end()); + DOCTEST_CHECK(first_vote->second == vote_for); + + const auto second_vote = votes.find(second_voter_id); + DOCTEST_CHECK(second_vote != votes.end()); + DOCTEST_CHECK(second_vote->second == vote_for); + } +} + +DOCTEST_TEST_CASE("User data") +{ + NetworkState network; + network.tables->set_encryptor(encryptor); + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + const auto member_id = gen.add_member(member_cert, {}); + const auto user_id = gen.add_user(user_cert); + set_whitelists(gen); + gen.set_gov_scripts(lua::Interpreter().invoke(gov_script_file)); + gen.finalize(); + + ShareManager share_manager(network); + StubNodeState node(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + const auto read_user_info = + create_request(read_params(user_id, Tables::USERS), "read"); + + { + DOCTEST_INFO("user data is initially empty"); + const auto read_response = parse_response_body( + frontend_process(frontend, read_user_info, member_cert)); + DOCTEST_CHECK(read_response.user_data.is_null()); + } + + { + auto user_data_object = nlohmann::json::object(); + user_data_object["name"] = "bob"; + user_data_object["permissions"] = {"read", "delete"}; + + DOCTEST_INFO("user data can be set to an object"); + Propose::In proposal; + proposal.script = fmt::format( + R"xxx( + proposed_user_data = {{ + name = "bob", + permissions = {{"read", "delete"}} + }} + return Calls:call("set_user_data", {{user_id = {}, user_data = + proposed_user_data}}) + )xxx", + user_id); + const auto proposal_serialized = + create_signed_request(proposal, "propose", kp); + const auto propose_response = parse_response_body( + frontend_process(frontend, proposal_serialized, member_cert)); + DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); + + DOCTEST_INFO("user data object can be read"); + const auto read_response = parse_response_body( + frontend_process(frontend, read_user_info, member_cert)); + DOCTEST_CHECK(read_response.user_data == user_data_object); + } + + { + const auto user_data_string = "ADMINISTRATOR"; + + DOCTEST_INFO("user data can be overwritten"); + Propose::In proposal; + proposal.script = std::string(R"xxx( + local tables, param = ... + return Calls:call("set_user_data", {user_id = param.id, user_data = + param.data}) + )xxx"); + proposal.parameter["id"] = user_id; + proposal.parameter["data"] = user_data_string; + const auto proposal_serialized = + create_signed_request(proposal, "propose", kp); + const auto propose_response = parse_response_body( + frontend_process(frontend, proposal_serialized, member_cert)); + DOCTEST_CHECK(propose_response.state == ProposalState::ACCEPTED); + + DOCTEST_INFO("user data object can be read"); + const auto response = parse_response_body( + frontend_process(frontend, read_user_info, member_cert)); + DOCTEST_CHECK(response.user_data == user_data_string); + } +} DOCTEST_TEST_CASE("Submit recovery shares") { @@ -1696,85 +1716,93 @@ DOCTEST_TEST_CASE("Submit recovery shares") } } -// DOCTEST_TEST_CASE("Maximum number of active members") -// { -// logger::config::level() = logger::INFO; - -// NetworkTables network; -// network.tables->set_encryptor(encryptor); -// auto node = StubNodeState(std::make_shared(network)); -// MemberRpcFrontend frontend(network, node); -// frontend.open(); - -// DOCTEST_INFO("Service is opening"); -// { -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); - -// for (size_t i = 0; i < max_active_members_count + 1; i++) -// { -// auto cert = get_cert_data(i, kp); -// if (i == max_active_members_count) -// { -// DOCTEST_REQUIRE_THROWS_AS_MESSAGE( -// gen.add_member(cert, {}), -// std::logic_error, -// fmt::format( -// "No more than {} active members are allowed", -// max_active_members_count)); -// } -// else -// { -// gen.add_member(cert, {}); -// } -// } -// } - -// DOCTEST_INFO("Service is open"); -// { -// std::map members; - -// Store::Tx gen_tx; -// GenesisGenerator gen(network, gen_tx); -// gen.init_values(); -// gen.create_service({}); -// gen.open_service(); -// gen.set_recovery_threshold(1); - -// // Service is open so members are added as ACCEPTED -// for (size_t i = 0; i < max_active_members_count + 1; i++) -// { -// auto cert = get_cert_data(i, kp); -// members[gen.add_member(cert, {})] = cert; -// } -// gen.finalize(); - -// for (auto const& m : members) -// { -// const auto state_digest_req = -// create_request(nullptr, "updateAckStateDigest"); -// const auto ack = parse_response_body( -// frontend_process(frontend, state_digest_req, m.second)); - -// StateDigest params; -// params.state_digest = ack.state_digest; -// const auto ack_req = create_signed_request(params, "ack", kp); -// const auto resp = frontend_process(frontend, ack_req, m.second); - -// if (m.first >= max_active_members_count) -// { -// DOCTEST_CHECK(resp.status == HTTP_STATUS_FORBIDDEN); -// } -// else -// { -// DOCTEST_CHECK(resp.status == HTTP_STATUS_OK); -// DOCTEST_CHECK(parse_response_body(resp)); -// } -// } -// } -// } +DOCTEST_TEST_CASE("Maximum number of active members") +{ + logger::config::level() = logger::INFO; + + NetworkState network; + network.ledger_secrets = std::make_shared(); + network.ledger_secrets->init(); + network.tables->set_encryptor(encryptor); + ShareManager share_manager(network); + StubNodeState node(share_manager); + MemberRpcFrontend frontend(network, node, share_manager); + frontend.open(); + + // DOCTEST_INFO("Service is opening"); + // { + // Store::Tx gen_tx; + // GenesisGenerator gen(network, gen_tx); + // gen.init_values(); + // gen.create_service({}); + // gen.set_recovery_threshold(1); + + // for (size_t i = 0; i < max_active_members_count + 1; i++) + // { + // auto cert = get_cert_data(i, kp); + // if (i == max_active_members_count) + // { + // DOCTEST_REQUIRE_THROWS_AS_MESSAGE( + // gen.add_member(cert, gen_public_encryption_key()), + // std::logic_error, + // fmt::format( + // "No more than {} active members are allowed", + // max_active_members_count)); + // } + // else + // { + // gen.add_member(cert, gen_public_encryption_key()); + // } + // } + // } + + DOCTEST_INFO("Service is open"); + { + std::map members; + + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + // Add one member as ACTIVE so that recovery threshold can be set to 1 + auto member0_cert = get_cert_data(0, kp); + members[gen.add_member(member0_cert, {})] = member0_cert; + gen.set_recovery_threshold(1); + gen.open_service(); + + // Service is open so members are added as ACCEPTED + for (size_t i = 1; i < max_active_members_count + 1; i++) + { + auto cert = get_cert_data(i, kp); + members[gen.add_member(cert, {})] = cert; + } + gen.finalize(); + + for (auto const& m : members) + { + const auto state_digest_req = + create_request(nullptr, "updateAckStateDigest"); + const auto ack = parse_response_body( + frontend_process(frontend, state_digest_req, m.second)); + + StateDigest params; + params.state_digest = ack.state_digest; + const auto ack_req = create_signed_request(params, "ack", kp); + const auto resp = frontend_process(frontend, ack_req, m.second); + + if (m.first >= max_active_members_count) + { + DOCTEST_CHECK(resp.status == HTTP_STATUS_FORBIDDEN); + } + else + { + DOCTEST_CHECK(resp.status == HTTP_STATUS_OK); + DOCTEST_CHECK(parse_response_body(resp)); + } + break; + } + } +} // We need an explicit main to initialize kremlib and EverCrypt int main(int argc, char** argv) From ee62eef6a0a68c99845761b4ceb80d464ef4b81a Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 11:23:23 +0100 Subject: [PATCH 14/33] Even better --- src/node/rpc/test/member_voting_test.cpp | 52 ++++++++++++------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 4fd87718d5a5..4fe96f2cd81d 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -1729,32 +1729,32 @@ DOCTEST_TEST_CASE("Maximum number of active members") MemberRpcFrontend frontend(network, node, share_manager); frontend.open(); - // DOCTEST_INFO("Service is opening"); - // { - // Store::Tx gen_tx; - // GenesisGenerator gen(network, gen_tx); - // gen.init_values(); - // gen.create_service({}); - // gen.set_recovery_threshold(1); - - // for (size_t i = 0; i < max_active_members_count + 1; i++) - // { - // auto cert = get_cert_data(i, kp); - // if (i == max_active_members_count) - // { - // DOCTEST_REQUIRE_THROWS_AS_MESSAGE( - // gen.add_member(cert, gen_public_encryption_key()), - // std::logic_error, - // fmt::format( - // "No more than {} active members are allowed", - // max_active_members_count)); - // } - // else - // { - // gen.add_member(cert, gen_public_encryption_key()); - // } - // } - // } + DOCTEST_INFO("Service is opening"); + { + Store::Tx gen_tx; + GenesisGenerator gen(network, gen_tx); + gen.init_values(); + gen.create_service({}); + gen.set_recovery_threshold(1); + + for (size_t i = 0; i < max_active_members_count + 1; i++) + { + auto cert = get_cert_data(i, kp); + if (i == max_active_members_count) + { + DOCTEST_REQUIRE_THROWS_AS_MESSAGE( + gen.add_member(cert, gen_public_encryption_key()), + std::logic_error, + fmt::format( + "No more than {} active members are allowed", + max_active_members_count)); + } + else + { + gen.add_member(cert, gen_public_encryption_key()); + } + } + } DOCTEST_INFO("Service is open"); { From b5615aecc755d4ebddd7b70206c0e51327adb6c1 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 11:52:23 +0100 Subject: [PATCH 15/33] Frontend tests fix --- src/node/rpc/test/frontend_test.cpp | 24 ++++++++++++++++-------- src/node/rpc/test/member_voting_test.cpp | 8 ++++---- src/node/rpc/test/node_frontend_test.cpp | 6 ++++-- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/src/node/rpc/test/frontend_test.cpp b/src/node/rpc/test/frontend_test.cpp index 6789a369ad98..1abbd22f6dba 100644 --- a/src/node/rpc/test/frontend_test.cpp +++ b/src/node/rpc/test/frontend_test.cpp @@ -174,8 +174,11 @@ class TestExplicitCommitability : public SimpleUserRpcFrontend class TestMemberFrontend : public MemberRpcFrontend { public: - TestMemberFrontend(ccf::NetworkState& network, ccf::StubNodeState& node) : - MemberRpcFrontend(network, node) + TestMemberFrontend( + ccf::NetworkState& network, + ccf::StubNodeState& node, + ccf::ShareManager& share_manager) : + MemberRpcFrontend(network, node, share_manager) { open(); @@ -273,8 +276,11 @@ class TestForwardingMemberFrontEnd : public MemberRpcFrontend, { public: TestForwardingMemberFrontEnd( - Store& tables, ccf::NetworkState& network, ccf::StubNodeState& node) : - MemberRpcFrontend(network, node) + Store& tables, + ccf::NetworkState& network, + ccf::StubNodeState& node, + ccf::ShareManager& share_manager) : + MemberRpcFrontend(network, node, share_manager) { open(); @@ -292,6 +298,7 @@ class TestForwardingMemberFrontEnd : public MemberRpcFrontend, auto kp = tls::make_key_pair(); NetworkState network; NetworkState network2; + auto encryptor = std::make_shared(); NetworkState pbft_network(ConsensusType::PBFT); @@ -304,7 +311,8 @@ auto history = std::make_shared( pbft_network.signatures, pbft_network.nodes); -StubNodeState stub_node; +ShareManager share_manager(network); +StubNodeState stub_node(share_manager); auto create_simple_request( const std::string& method = "empty_function", @@ -729,7 +737,7 @@ TEST_CASE("Member caller") prepare_callers(); auto simple_call = create_simple_request(); std::vector serialized_call = simple_call.build_request(); - TestMemberFrontend frontend(network, stub_node); + TestMemberFrontend frontend(network, stub_node, share_manager); SUBCASE("valid caller") { @@ -1346,9 +1354,9 @@ TEST_CASE("Memberfrontend forwarding" * doctest::test_suite("forwarding")) add_callers_primary_store(); TestForwardingMemberFrontEnd member_frontend_backup( - *network.tables, network, stub_node); + *network.tables, network, stub_node, share_manager); TestForwardingMemberFrontEnd member_frontend_primary( - *network2.tables, network2, stub_node); + *network2.tables, network2, stub_node, share_manager); auto channel_stub = std::make_shared(); auto backup_forwarder = std::make_shared>( diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 4fe96f2cd81d..7567fae7101a 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -64,13 +64,11 @@ T parse_response_body(const TResponse& r) try { body_j = jsonrpc::unpack(r.body, jsonrpc::Pack::Text); - LOG_FAIL_FMT("RPC resp: {}", body_j.dump()); } catch (const nlohmann::json::parse_error& e) { - std::cerr << e.what() << std::endl; - std::cerr << "RPC error: " << std::string(r.body.begin(), r.body.end()) - << std::endl; + LOG_FAIL_FMT(e.what()); + LOG_FAIL_FMT("RPC error: {}", std::string(r.body.begin(), r.body.end())); } return body_j.get(); @@ -318,6 +316,7 @@ DOCTEST_TEST_CASE("Member query/read") } } + DOCTEST_TEST_CASE("Proposer ballot") { NetworkState network; @@ -1804,6 +1803,7 @@ DOCTEST_TEST_CASE("Maximum number of active members") } } + // We need an explicit main to initialize kremlib and EverCrypt int main(int argc, char** argv) { diff --git a/src/node/rpc/test/node_frontend_test.cpp b/src/node/rpc/test/node_frontend_test.cpp index e124d88a9de2..061e9bbf2a5b 100644 --- a/src/node/rpc/test/node_frontend_test.cpp +++ b/src/node/rpc/test/node_frontend_test.cpp @@ -80,7 +80,8 @@ TEST_CASE("Add a node to an opening service") GenesisGenerator gen(network, gen_tx); gen.init_values(); - StubNodeState node; + ShareManager share_manager(network); + StubNodeState node(share_manager); NodeRpcFrontend frontend(network, node); frontend.open(); @@ -200,7 +201,8 @@ TEST_CASE("Add a node to an open service") GenesisGenerator gen(network, gen_tx); gen.init_values(); - StubNodeState node; + ShareManager share_manager(network); + StubNodeState node(share_manager); node.set_is_public(true); NodeRpcFrontend frontend(network, node); frontend.open(); From c35575b1af09b1c43ae1fdfe19353e5a52e2957a Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 16:10:24 +0100 Subject: [PATCH 16/33] Unit test for bogus recovery shares --- src/node/node_state.h | 29 ++++++++++---------- src/node/rpc/member_frontend.h | 14 +++++++++- src/node/rpc/test/member_voting_test.cpp | 34 ++++++++++++++++++++++-- src/node/share_manager.h | 13 ++++++--- 4 files changed, 68 insertions(+), 22 deletions(-) diff --git a/src/node/node_state.h b/src/node/node_state.h index 559695c28895..80fceb1f7395 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -734,6 +734,7 @@ namespace ccf // Open the service if (consensus->is_primary()) { + LOG_FAIL_FMT("Is primary!"); Store::Tx tx; // Shares for the new ledger secret can only be issued now, once the @@ -886,7 +887,8 @@ namespace ccf { if ( !sm.check(State::partOfNetwork) && - !sm.check(State::partOfPublicNetwork)) + !sm.check(State::partOfPublicNetwork) && + !sm.check(State::readingPrivateLedger)) return; consensus->periodic_end(); @@ -929,7 +931,8 @@ namespace ccf { return ( (sm.check(State::partOfNetwork) || - sm.check(State::partOfPublicNetwork)) && + sm.check(State::partOfPublicNetwork) || + sm.check(State::readingPrivateLedger)) && consensus->is_primary()); } @@ -1029,20 +1032,12 @@ namespace ccf void restore_ledger_secrets(Store::Tx& tx) override { - try - { - finish_recovery( - tx, - share_manager.restore_recovery_shares_info( - tx, recovery_ledger_secrets)); + finish_recovery( + tx, + share_manager.restore_recovery_shares_info( + tx, recovery_ledger_secrets)); - recovery_ledger_secrets.clear(); - } - catch (const std::logic_error& e) - { - throw std::logic_error( - fmt::format("Failed to restore recovery shares info: {}", e.what())); - } + recovery_ledger_secrets.clear(); } NodeId get_node_id() const override @@ -1135,6 +1130,7 @@ namespace ccf for (auto [nid, ni] : trusted_nodes) { + LOG_FAIL_FMT("Broadcast secret to node {}", nid); ccf::EncryptedLedgerSecret secret_for_node; secret_for_node.node_id = nid; @@ -1354,10 +1350,13 @@ namespace ccf bool has_secrets = false; std::list restored_secrets; + LOG_FAIL_FMT("Secrets hook!"); + for (auto& [v, secret_set] : w) { for (auto& encrypted_secret_for_node : secret_set.value.secrets) { + LOG_FAIL_FMT("For node {}", encrypted_secret_for_node.node_id); if (encrypted_secret_for_node.node_id == self) { crypto::GcmCipher gcmcipher; diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 1b8ef3967694..194f03d4d53e 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -932,7 +932,19 @@ namespace ccf LOG_DEBUG_FMT( "Reached secret sharing threshold {}", g.get_recovery_threshold()); - node.restore_ledger_secrets(args.tx); + try + { + node.restore_ledger_secrets(args.tx); + } + catch (const std::logic_error& e) + { + // For now, clear the submitted shares if combination fails. + share_manager.clear_submitted_recovery_shares(args.tx); + return make_error( + HTTP_STATUS_INTERNAL_SERVER_ERROR, + fmt::format( + "Failed to restore recovery shares info: {}", e.what())); + } share_manager.clear_submitted_recovery_shares(args.tx); return make_success(true); diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 7567fae7101a..2cf548dbe990 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -316,7 +316,6 @@ DOCTEST_TEST_CASE("Member query/read") } } - DOCTEST_TEST_CASE("Proposer ballot") { NetworkState network; @@ -1690,6 +1689,38 @@ DOCTEST_TEST_CASE("Submit recovery shares") DOCTEST_REQUIRE_FALSE(g.set_recovery_threshold(recovery_threshold)); } + DOCTEST_INFO("Submit bogus recovery shares"); + { + size_t submitted_shares_count = 0; + for (auto const& m : members) + { + auto bogus_recovery_share = retrieved_shares[m.first]; + bogus_recovery_share[0] = bogus_recovery_share[0] + 1; + const auto submit_recovery_share = create_request( + SubmitRecoveryShare({bogus_recovery_share}), "submitRecoveryShare"); + + auto rep = + frontend_process(frontend, submit_recovery_share, m.second.first); + + submitted_shares_count++; + + // Share submission should only complete when the recovery threshold + // has been reached + if (submitted_shares_count < recovery_threshold) + { + DOCTEST_REQUIRE(parse_response_body(rep) == false); + } + else + { + check_error(rep, HTTP_STATUS_INTERNAL_SERVER_ERROR); + break; + } + } + } + + // It is still possible to re-submit recovery shares if a threshold of at + // least one bogus share has been submitted. + DOCTEST_INFO("Submit recovery shares"); { size_t submitted_shares_count = 0; @@ -1803,7 +1834,6 @@ DOCTEST_TEST_CASE("Maximum number of active members") } } - // We need an explicit main to initialize kremlib and EverCrypt int main(int argc, char** argv) { diff --git a/src/node/share_manager.h b/src/node/share_manager.h index df042a056b13..dd4403236a3f 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -399,12 +399,10 @@ namespace ccf [&submitted_shares_count]( const MemberId member_id, const std::vector& encrypted_share) { - LOG_FAIL_FMT("Submitted share for member {}", member_id); submitted_shares_count++; return true; }); - LOG_FAIL_FMT("submitted_shares_count: {}", submitted_shares_count); return submitted_shares_count; } @@ -412,13 +410,20 @@ namespace ccf { auto submitted_shares_view = tx.get_view(network.submitted_shares); + std::vector submitted_share_ids = {}; + submitted_shares_view->foreach( - [&submitted_shares_view]( + [&submitted_share_ids]( const MemberId member_id, const std::vector& encrypted_share) { - submitted_shares_view->remove(member_id); + submitted_share_ids.push_back(member_id); return true; }); + + for (auto const& id : submitted_share_ids) + { + submitted_shares_view->remove(id); + } } }; } \ No newline at end of file From 0e5981ee7b225ff51bde8357664d83982829cc7f Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 16:19:12 +0100 Subject: [PATCH 17/33] First cleanup before PR --- CMakeLists.txt | 2 +- src/node/node_state.h | 8 +------- tests/infra/ccf.py | 3 --- tests/recovery.py | 10 +++++----- 4 files changed, 7 insertions(+), 16 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index acb896cb1d4e..c4ecd6a79bc4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -461,7 +461,7 @@ if(BUILD_TESTS) NAME recovery_test PYTHON_SCRIPT ${CMAKE_SOURCE_DIR}/tests/recovery.py CONSENSUS raft - ADDITIONAL_ARGS --recovery 2 + ADDITIONAL_ARGS --recovery 2 --raft-election-timeout 4000 ) add_e2e_test( diff --git a/src/node/node_state.h b/src/node/node_state.h index 80fceb1f7395..6b7ac7abc7c0 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -734,7 +734,6 @@ namespace ccf // Open the service if (consensus->is_primary()) { - LOG_FAIL_FMT("Is primary!"); Store::Tx tx; // Shares for the new ledger secret can only be issued now, once the @@ -887,8 +886,7 @@ namespace ccf { if ( !sm.check(State::partOfNetwork) && - !sm.check(State::partOfPublicNetwork) && - !sm.check(State::readingPrivateLedger)) + !sm.check(State::partOfPublicNetwork)) return; consensus->periodic_end(); @@ -1130,7 +1128,6 @@ namespace ccf for (auto [nid, ni] : trusted_nodes) { - LOG_FAIL_FMT("Broadcast secret to node {}", nid); ccf::EncryptedLedgerSecret secret_for_node; secret_for_node.node_id = nid; @@ -1350,13 +1347,10 @@ namespace ccf bool has_secrets = false; std::list restored_secrets; - LOG_FAIL_FMT("Secrets hook!"); - for (auto& [v, secret_set] : w) { for (auto& encrypted_secret_for_node : secret_set.value.secrets) { - LOG_FAIL_FMT("For node {}", encrypted_secret_for_node.node_id); if (encrypted_secret_for_node.node_id == self) { crypto::GcmCipher gcmcipher; diff --git a/tests/infra/ccf.py b/tests/infra/ccf.py index 2a7b28353b81..686413d61045 100644 --- a/tests/infra/ccf.py +++ b/tests/infra/ccf.py @@ -342,9 +342,6 @@ def recover(self, args, defunct_network_enc_pub): :param args: command line arguments to configure the CCF nodes. :param defunct_network_enc_pub: defunct network encryption public key. """ - # if defunct_network_enc_pub is None: - # defunct_network_enc_pub = self.store_current_network_encryption_key() - primary, _ = self.find_primary() self.consortium.check_for_service(primary, status=ServiceStatus.OPENING) self.consortium.wait_for_all_nodes_to_be_trusted(primary, self.nodes) diff --git a/tests/recovery.py b/tests/recovery.py index 114c603a7fbb..21ab84ab6604 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -95,7 +95,7 @@ def test_share_resilience(network, args): def run(args): - hosts = ["localhost", "localhost", "localhost"] + hosts = ["localhost", "localhost"] txs = app.LoggingTxs() @@ -106,10 +106,10 @@ def run(args): for i in range(args.recovery): # Alternate between recovery with primary change and stable primary-ship - # if i % 2 == 0: - recovered_network = test_share_resilience(network, args) - # else: - # recovered_network = test(network, args) + if i % 2 == 0: + recovered_network = test_share_resilience(network, args) + else: + recovered_network = test(network, args) network.stop_all_nodes() network = recovered_network LOG.success("Recovery complete on all nodes") From f1b41c51a95a4259b7dd0fbb48ab76f39e0f0d85 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 16:33:05 +0100 Subject: [PATCH 18/33] Better with three nodes --- tests/recovery.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/recovery.py b/tests/recovery.py index 21ab84ab6604..1230a21940ec 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -68,6 +68,9 @@ def test_share_resilience(network, args): # Here, we kill the current primary instead of just suspending it. # However, because of https://github.com/microsoft/CCF/issues/99#issuecomment-630875387, # the new primary will most likely be the previous primary, which defies the point of this test. + LOG.info( + f"Shutting down node {primary.node_id} before submitting last recovery share" + ) primary.stop() LOG.debug( f"Waiting {recovered_network.election_duration}s for a new primary to be elected..." @@ -95,7 +98,7 @@ def test_share_resilience(network, args): def run(args): - hosts = ["localhost", "localhost"] + hosts = ["localhost", "localhost", "localhost"] txs = app.LoggingTxs() From 60938a6c8342d2b4f9c65f494bab38965352a41c Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 16:47:40 +0100 Subject: [PATCH 19/33] Merge conflicts --- src/node/rpc/test/node_stub.h | 2 +- src/node/submitted_shares.h | 5 +++-- tests/infra/checker.py | 12 ++---------- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/node/rpc/test/node_stub.h b/src/node/rpc/test/node_stub.h index 61e769029739..e9d26f87fbf4 100644 --- a/src/node/rpc/test/node_stub.h +++ b/src/node/rpc/test/node_stub.h @@ -62,7 +62,7 @@ namespace ccf const std::optional>& filter) override {} - void restore_ledger_secrets(Store::Tx& tx) override + void restore_ledger_secrets(kv::Tx& tx) override { share_manager.restore_recovery_shares_info(tx, {}); } diff --git a/src/node/submitted_shares.h b/src/node/submitted_shares.h index b54441a3d878..205d1ae29292 100644 --- a/src/node/submitted_shares.h +++ b/src/node/submitted_shares.h @@ -3,8 +3,9 @@ #pragma once #include "entities.h" +#include "kv/map.h" -#include +#include namespace ccf { @@ -14,5 +15,5 @@ namespace ccf // Because shares are submitted to the public-only network on recovery, this // table is public but the shares are encrypted with the latest ledger secret. - using SubmittedShares = Store::Map>; + using SubmittedShares = kv::Map>; } \ No newline at end of file diff --git a/tests/infra/checker.py b/tests/infra/checker.py index c2a6875bd090..fdb47e49ebb2 100644 --- a/tests/infra/checker.py +++ b/tests/infra/checker.py @@ -8,11 +8,7 @@ from infra.tx_status import TxStatus -<<<<<<< HEAD -def wait_for_global_commit(client, commit_index, term, mksign=False, timeout=3): -======= -def wait_for_global_commit(node_client, seqno, view, mksign=False, timeout=3): ->>>>>>> upstream/master +def wait_for_global_commit(client, seqno, view, mksign=False, timeout=3): """ Given a client to a CCF network and a seqno/view pair, this function waits for this specific commit index to be globally committed by the @@ -31,11 +27,7 @@ def wait_for_global_commit(node_client, seqno, view, mksign=False, timeout=3): end_time = time.time() + timeout while time.time() < end_time: -<<<<<<< HEAD - r = client.get("tx", {"view": term, "seqno": commit_index}) -======= - r = node_client.get("tx", {"view": view, "seqno": seqno}) ->>>>>>> upstream/master + r = client.get("tx", {"view": view, "seqno": seqno}) assert ( r.status == http.HTTPStatus.OK ), f"tx request returned HTTP status {r.status}" From c59f2491c576195101aa87c780f9acd68738f6ee Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 16:51:20 +0100 Subject: [PATCH 20/33] Finding a primary may take a long time when recovering the public ledger --- tests/infra/ccf.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/infra/ccf.py b/tests/infra/ccf.py index 5a99093013f0..3dcc69270f1f 100644 --- a/tests/infra/ccf.py +++ b/tests/infra/ccf.py @@ -151,14 +151,16 @@ def create_node(self, host): self.nodes.append(node) return node - def _add_node(self, node, lib_name, args, target_node=None): + def _add_node(self, node, lib_name, args, target_node=None, recovery=False): forwarded_args = { arg: getattr(args, arg) for arg in infra.ccf.Network.node_args_to_forward } # Contact primary if no target node is set if target_node is None: - target_node, _ = self.find_primary() + target_node, _ = self.find_primary( + timeout=args.ledger_recovery_timeout if recovery else 3 + ) node.join( lib_name=lib_name, @@ -225,7 +227,7 @@ def _start_all_nodes(self, args, recovery=False, ledger_file=None): ) self._adjust_local_node_ids(node) else: - self._add_node(node, args.package, args) + self._add_node(node, args.package, args, recovery=recovery) except Exception: LOG.exception("Failed to start node {}".format(node.node_id)) raise From 66c2e4881813838078e56e7bd026c1903086c984 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Fri, 22 May 2020 17:03:37 +0100 Subject: [PATCH 21/33] Fix before PR --- src/node/network_tables.h | 4 +--- src/node/node_state.h | 6 ++---- tests/infra/ccf.py | 1 + tests/infra/checker.py | 4 +--- tests/infra/member.py | 2 -- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/src/node/network_tables.h b/src/node/network_tables.h index 63038b63e19e..a3a2ea290768 100644 --- a/src/node/network_tables.h +++ b/src/node/network_tables.h @@ -120,9 +120,7 @@ namespace ccf shares( tables->create(Tables::SHARES, kv::SecurityDomain::PUBLIC)), submitted_shares(tables->create( - Tables::SUBMITTED_SHARES, - kv::SecurityDomain::PUBLIC)), // TODO: Submitted shares should not be - // public!!! + Tables::SUBMITTED_SHARES, kv::SecurityDomain::PUBLIC)), users(tables->create(Tables::USERS)), config(tables->create( Tables::CONFIGURATION, kv::SecurityDomain::PUBLIC)), diff --git a/src/node/node_state.h b/src/node/node_state.h index 8e8e9820bdbc..b6a7189f73b6 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -933,10 +933,8 @@ namespace ccf bool is_primary() const override { return ( - (sm.check(State::partOfNetwork) || - sm.check(State::partOfPublicNetwork) || - sm.check(State::readingPrivateLedger)) && - consensus->is_primary()); + sm.check(State::partOfNetwork) && + sm.check(State::partOfPublicNetwork) && consensus->is_primary()); } bool is_part_of_network() const override diff --git a/tests/infra/ccf.py b/tests/infra/ccf.py index 3dcc69270f1f..001efe5220d6 100644 --- a/tests/infra/ccf.py +++ b/tests/infra/ccf.py @@ -109,6 +109,7 @@ def __init__( self.status = ServiceStatus.CLOSED self.binary_dir = binary_dir self.common_dir = None + self.election_duration = None self.key_generator = os.path.join(binary_dir, self.KEY_GEN) if not os.path.isfile(self.key_generator): raise FileNotFoundError( diff --git a/tests/infra/checker.py b/tests/infra/checker.py index fdb47e49ebb2..86a8393b177a 100644 --- a/tests/infra/checker.py +++ b/tests/infra/checker.py @@ -70,9 +70,7 @@ def __call__(self, rpc_result, result=None, error=None, timeout=2): ) if self.client: - wait_for_global_commit( - self.client, rpc_result.seqno, rpc_result.view - ) + wait_for_global_commit(self.client, rpc_result.seqno, rpc_result.view) if self.notification_queue: end_time = time.time() + timeout diff --git a/tests/infra/member.py b/tests/infra/member.py index 6154a377cc75..a0bc30b1e91b 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -9,8 +9,6 @@ import http import os -from loguru import logger as LOG - class NoRecoveryShareFound(Exception): def __init__(self, response): From e6a7a32e06eb19751c0b12c585e99c52b93749f4 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Sun, 24 May 2020 21:03:28 +0100 Subject: [PATCH 22/33] Remove TODO --- src/node/share_manager.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/node/share_manager.h b/src/node/share_manager.h index ef3f4209658e..3b9155787605 100644 --- a/src/node/share_manager.h +++ b/src/node/share_manager.h @@ -186,7 +186,6 @@ namespace ccf compute_encrypted_shares(tx, ls_wrapping_key)}); } - // TODO: Move this to a different class? std::vector encrypt_submitted_share( const std::vector& submitted_share) { From c2364a7c8b285bb1d915226801bc7ffa6bfe5068 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 26 May 2020 09:07:29 +0100 Subject: [PATCH 23/33] WIP --- cmake/common.cmake | 5 ++- tests/infra/ccf.py | 5 ++- tests/infra/consortium.py | 37 ++++++++++------ tests/infra/member.py | 66 +++++++++++++++------------- tests/memberclient.py | 3 +- tests/recovery.py | 13 ++++-- tests/submit_recovery_share.sh | 79 +++++++--------------------------- 7 files changed, 93 insertions(+), 115 deletions(-) diff --git a/cmake/common.cmake b/cmake/common.cmake index 29a1e7b3d31f..504c47a87221 100644 --- a/cmake/common.cmake +++ b/cmake/common.cmake @@ -91,7 +91,7 @@ add_custom_command( # Copy utilities from tests directory set(CCF_UTILITIES tests.sh keygenerator.sh cimetrics_env.sh - upload_pico_metrics.py scurl.sh + upload_pico_metrics.py scurl.sh submit_recovery_share.sh ) foreach(UTILITY ${CCF_UTILITIES}) configure_file( @@ -101,7 +101,8 @@ endforeach() # Install specific utilities install(PROGRAMS ${CCF_DIR}/tests/scurl.sh ${CCF_DIR}/tests/keygenerator.sh - ${CCF_DIR}/tests/sgxinfo.sh DESTINATION bin + ${CCF_DIR}/tests/sgxinfo.sh + ${CCF_DIR}/tests/submit_recovery_share.sh DESTINATION bin ) # Install getting_started scripts for VM creation and setup diff --git a/tests/infra/ccf.py b/tests/infra/ccf.py index 9fe1d8473ae8..8e3f62d634af 100644 --- a/tests/infra/ccf.py +++ b/tests/infra/ccf.py @@ -57,6 +57,7 @@ def get_common_folder_name(workspace, label): class Network: KEY_GEN = "keygenerator.sh" + SHARE_SCRIPT = "submit_recovery_share.sh" DEFUNCT_NETWORK_ENC_PUBK = "network_enc_pubk_orig.pem" node_args_to_forward = [ "enclave_type", @@ -111,6 +112,7 @@ def __init__( self.common_dir = None self.election_duration = None self.key_generator = os.path.join(binary_dir, self.KEY_GEN) + self.share_script = os.path.join(binary_dir, self.SHARE_SCRIPT) if not os.path.isfile(self.key_generator): raise FileNotFoundError( f"Could not find key generator script at '{self.key_generator}' - is binary directory set correctly?" @@ -277,6 +279,7 @@ def start_and_join(self, args): self.consortium = infra.consortium.Consortium( self.common_dir, self.key_generator, + self.share_script, initial_member_ids, args.participants_curve, ) @@ -329,7 +332,7 @@ def start_in_recovery( # If a common directory was passed in, initialise the consortium from it if common_dir is not None: self.consortium = infra.consortium.Consortium( - common_dir, self.key_generator, remote_node=primary + common_dir, self.key_generator, self.share_script, remote_node=primary ) for node in self.get_joined_nodes(): diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index f1349cdaee13..565a2d7505c3 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -19,18 +19,27 @@ class Consortium: def __init__( - self, common_dir, key_generator, member_ids=None, curve=None, remote_node=None + self, + common_dir, + key_generator, + share_script, + member_ids=None, + curve=None, + remote_node=None, ): self.common_dir = common_dir self.members = [] self.key_generator = key_generator + self.share_script = share_script self.members = [] self.recovery_threshold = None # If a list of member IDs is passed in, generate fresh member identities. # Otherwise, recover the state of the consortium from the state of CCF. if member_ids is not None: for m_id in member_ids: - new_member = infra.member.Member(m_id, curve, common_dir, key_generator) + new_member = infra.member.Member( + m_id, curve, common_dir, share_script, key_generator + ) new_member.set_active() self.members.append(new_member) self.recovery_threshold = len(self.members) @@ -51,7 +60,9 @@ def __init__( }, ) for m in r.result or []: - new_member = infra.member.Member(m[0], curve, self.common_dir) + new_member = infra.member.Member( + m[0], curve, self.common_dir, share_script + ) if ( infra.member.MemberStatus[m[1]] == infra.member.MemberStatus.ACTIVE @@ -75,7 +86,7 @@ def generate_and_propose_new_member(self, remote_node, curve): # should ACK to become active. new_member_id = len(self.members) new_member = infra.member.Member( - new_member_id, curve, self.common_dir, self.key_generator + new_member_id, curve, self.common_dir, self.share_script, self.key_generator ) script = """ @@ -319,17 +330,15 @@ def recover_with_shares(self, remote_node, defunct_network_enc_pubk): check_commit = infra.checker.Checker(nc) for m in self.get_active_members(): - decrypted_share = m.get_and_decrypt_recovery_share( - remote_node, defunct_network_enc_pubk - ) - r = m.submit_recovery_share(remote_node, decrypted_share) + m.get_and_submit_recovery_share(remote_node, defunct_network_enc_pubk) + # r = m.submit_recovery_share(remote_node, decrypted_share) submitted_shares_count += 1 - check_commit( - r, - result=True - if submitted_shares_count >= self.recovery_threshold - else False, - ) + # check_commit( + # r, + # result=True + # if submitted_shares_count >= self.recovery_threshold + # else False, + # ) if submitted_shares_count >= self.recovery_threshold: break diff --git a/tests/infra/member.py b/tests/infra/member.py index 98dd7d5426d9..3b360357d66f 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -26,17 +26,17 @@ class MemberStatus(Enum): class Member: - def __init__(self, member_id, curve, common_dir, key_generator=None): - self.key_generator = key_generator + def __init__(self, member_id, curve, common_dir, share_script, key_generator=None): self.common_dir = common_dir self.member_id = member_id self.status = MemberStatus.ACCEPTED + self.share_script = share_script if key_generator is not None: # For now, all members are given an encryption key (for recovery) member = f"member{member_id}" infra.proc.ccall( - self.key_generator, + key_generator, "--name", f"{member}", "--curve", @@ -141,36 +141,42 @@ def ack(self, remote_node): self.status = MemberStatus.ACTIVE def get_and_decrypt_recovery_share(self, remote_node, defunct_network_enc_pubk): - LOG.warning( - f"About to retrieve and decrypt recovery share... {remote_node.host}:{remote_node.rpc_port}" - ) - while True: - pass - with remote_node.member_client(member_id=self.member_id) as mc: - r = mc.get("getEncryptedRecoveryShare") - if r.status != http.HTTPStatus.OK.value: - raise NoRecoveryShareFound(r) + r = mc.rpc("getEncryptedRecoveryShare", params={}) - # Members rely on a copy of the original network encryption public - # key to decrypt their recovery shares ctx = infra.crypto.CryptoBoxCtx( - os.path.join(self.common_dir, f"member{self.member_id}_enc_privk.pem"), + os.path.join(self.common_dir, f"member{self.member_id}_enc_priv.pem"), defunct_network_enc_pubk, ) - return base64.b64encode( - ctx.decrypt( - base64.b64decode(r.result["encrypted_recovery_share"]), - base64.b64decode(r.result["nonce"]), - ) - ).decode() - - def submit_recovery_share(self, remote_node, decrypted_recovery_share): - with remote_node.member_client(member_id=self.member_id) as mc: - r = mc.rpc( - "submitRecoveryShare", - params={"recovery_share": decrypted_recovery_share}, - ) - assert r.error is None, f"Error submitting recovery share: {r.error}" - return r + nonce_bytes = bytes(r.result["nonce"]) + encrypted_share_bytes = bytes(r.result["encrypted_share"]) + return ctx.decrypt(encrypted_share_bytes, nonce_bytes) + + def get_and_submit_recovery_share(self, remote_node, defunct_network_enc_pubk): + # For now, all members are given an encryption key (for recovery) + infra.proc.ccall( + self.share_script, + "--rpc-address", + f"{remote_node.host}:{remote_node.rpc_port}", + "--member-enc-privk", + os.path.join(self.common_dir, f"member{self.member_id}_enc_privk.pem"), + "--network-enc-pubk", + defunct_network_enc_pubk, + "--cert", + os.path.join(self.common_dir, f"member{self.member_id}_cert.pem"), + "--key", + os.path.join(self.common_dir, f"member{self.member_id}_privk.pem"), + "--cacert", + os.path.join(self.common_dir, "networkcert.pem"), + "-i", + ).check_returncode() + + # def submit_recovery_share(self, remote_node, decrypted_recovery_share): + # with remote_node.member_client(member_id=self.member_id) as mc: + # r = mc.rpc( + # "submitRecoveryShare", + # params={"recovery_share": decrypted_recovery_share}, + # ) + # assert r.error is None, f"Error submitting recovery share: {r.error}" + # return r diff --git a/tests/memberclient.py b/tests/memberclient.py index d98317de554f..78aea5bd01e1 100644 --- a/tests/memberclient.py +++ b/tests/memberclient.py @@ -42,7 +42,7 @@ def test_add_member(network, args): ) try: - new_member.get_and_decrypt_recovery_share( + new_member.get_and_submit_recovery_share( primary, network.store_current_network_encryption_key() ) assert False, "New accepted members are not given recovery shares" @@ -81,6 +81,7 @@ def assert_recovery_shares_update(func, network, args, **kwargs): network.store_current_network_encryption_key() already_active_member = network.consortium.get_any_active_member() defunct_network_enc_pubk = network.store_current_network_encryption_key() + # TODO: Only get the encrypted share saved_share = already_active_member.get_and_decrypt_recovery_share( primary, defunct_network_enc_pubk ) diff --git a/tests/recovery.py b/tests/recovery.py index 1230a21940ec..51792d65a9b0 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -40,6 +40,7 @@ def test_share_resilience(network, args): # Submit all required recovery shares minus one. Last recovery share is # submitted after a new primary is found. + LOG.error(recovered_network.consortium.recovery_threshold) submitted_shares_count = 0 for m in recovered_network.consortium.get_active_members(): with primary.node_client() as nc: @@ -50,15 +51,19 @@ def test_share_resilience(network, args): last_member_to_submit = m break + # TODO: Address issue of knowing when share is globally committed? check_commit = infra.checker.Checker(nc) decrypted_share = m.get_and_decrypt_recovery_share( primary, defunct_network_enc_pubk ) - check_commit( - m.submit_recovery_share(primary, decrypted_share), result=False - ) + # check_commit( + # m.submit_recovery_share(primary, decrypted_share), result=False + # ) submitted_shares_count += 1 + import time + time.sleep(2) + # In theory, check_commit should be sufficient to guarantee that the new primary # will know about all the recovery shares submitted so far. However, because of # https://github.com/microsoft/CCF/issues/589, we have to wait for all nodes @@ -84,7 +89,7 @@ def test_share_resilience(network, args): decrypted_share = last_member_to_submit.get_and_decrypt_recovery_share( new_primary, defunct_network_enc_pubk ) - last_member_to_submit.submit_recovery_share(new_primary, decrypted_share) + # last_member_to_submit.submit_recovery_share(new_primary, decrypted_share) for node in recovered_network.get_joined_nodes(): recovered_network.wait_for_state( diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh index add21851c85e..93f8459c2a6c 100755 --- a/tests/submit_recovery_share.sh +++ b/tests/submit_recovery_share.sh @@ -4,11 +4,16 @@ set -e + +# TODO: Why is --rpc-address passed in? Cannot use https:// instead? +# Because of the method/verb! function usage() { echo "Usage:""" - echo " $0 --rpc-address=node_rpc_address --member-enc-privk=member_enc_privk.pem --network-enc-pubk=network_enc_pubk.pem [CURL_OPTIONS]" - echo "Retrieves the encrypted recovery share for a given member, decrypts the share and submit the share to initiate the end of the recovery protocol." + echo " $0 --rpc-address rpc_address --member-enc-privk member_enc_privk.pem --network-enc-pubk network_enc_pubk.pem [CURL_OPTIONS]" + echo "Retrieves the encrypted recovery share for a given member, decrypts the share and submits it for recovery." + echo "" + echo "A sufficient number of recovery shares must be submitted by members to initiate the end of recovery procedure." echo "Note: Requires step CLI." } @@ -57,80 +62,28 @@ fi # TODO: Check for errors, probably using || +# set -x # Retrieve encrypted recovery share and nonce resp=$(curl -sS https://${node_rpc_address}/members/getEncryptedRecoveryShare ${@}) encrypted_share="$(echo ${resp} | jq -r .encrypted_recovery_share)" nonce="$(echo ${resp} | jq -r .nonce)" -# GOOD -# encrypted_share="gITiie6qa28H7HccwsxfhhFOzK82d2o2dX9iT+ozK4P9PtMrisPehByE1g2bGzh1sCHuUwQYpv3nME/jIuG6qfj2AQDNZMHebA/BW5ZNvc4ZsxLV4uvtS6zkpQ0yj4JOUnmSRqm/5FSG2H/L+8m+wsX8PdMSBJfqsnaFewIsiwhW" - -# BAD -# encrypted_share="AJBL0MSkenXochLnUrW2SQ0Cb8kzapfULv7e4I+CO+tDQwDSxAh+TRN1" - -# # GOOD -# nonce="iCsAvndW5us2wTLZG70khtMrTlhP0OK4" - -# BAD -# nonce="NlkAqQvA2RLG+xWFSrw+JNP3yAy8Vq5Z" - - -echo "Encrypted recovery share: ${encrypted_share}" -echo "Nonce: ${nonce}" +# echo "Encrypted recovery share: ${encrypted_share}" +# echo "Nonce: ${nonce}" # Parse raw private key from SubjectPublicKeyInfo DER format, as generated by keygenerator.sh --gen-enc-key der_header_privk_len=14 openssl asn1parse -in ${member_enc_privk} -strparse ${der_header_privk_len} -out key.raw -noout -# Parse raw public key generated by network +# # Parse raw public key generated by network der_header_pubk_len=9 openssl asn1parse -in ${network_enc_pubk} -i -strparse ${der_header_pubk_len} -out key2.raw -noout -# printf "${nonce}" | base64 -d -# exit 0 - - -# set -x +# step Base64 standard to URL encoding +encrypted_share_b64_url=$(echo ${encrypted_share} | sed 's/+/-/g; s/\//_/g') # Decrypt encrypted share with nonce, member private key and previous network public key +decrypted_share=$(echo "${encrypted_share_b64_url}" | step crypto nacl box open base64:"${nonce}" key2.raw key.raw | openssl base64 -A) -# decrypted_share=$(echo -n "${encrypted_share}" | openssl base64 -d | step crypto nacl box open "$(echo -n ${nonce} | openssl base64 -d)" key2.raw key.raw -raw | openssl base64 -A) - -# echo ${nonce} | openssl base64 -d | xargs --null echo "${encrypted_share}" | openssl base64 -d | step crypto nacl box open '{}' key2.raw key.raw -raw - -# echo "$(echo -ne ${nonce} | openssl base64 -d)" - -# echo "" - -# Works: -# echo "bGFsYQo=" | openssl base64 -d | xargs -I{} sh -c "echo \"p91/FnEWQWWc1pWUc/dOuOFW6tB602A9\" | openssl base64 -d | step crypto nacl box open {} alice.pub bob.priv -raw" - -# echo "$nonce" | openssl base64 -d | xargs --null -I{} echo $"{encrypted_share}" | openssl base64 -d | step crypto nacl box open {} key2.raw key.raw -raw - - -# nonce="fi2rgnUTTRcvukMaMLYAkG7NScQ5q32e" -# var=$(echo -n "$nonce" | base64 -d | xargs --null -I {} sh -c "echo {} | hd") -# echo "var: $var" - -# # set -x -# decrypted_share1=$(echo -n ${nonce} | base64 -d | xargs --null -I '{}' sh -c "echo "${encrypted_share}" | base64 -d | step crypto nacl box open \"{}\" key2.raw key.raw -raw | openssl base64 -A") -# set +x - -# # echo ${nonce} | openssl base64 -d | xargs -I{} sh -c "echo {}" - -echo "Decrypted share: ${decrypted_share1}" - -# # Orign: -# decrypted_share=$(echo "${encrypted_share}" | openssl base64 -d | step crypto nacl box open "$(echo ${nonce} | openssl base64 -d)" key2.raw key.raw -raw | openssl base64 -A) - -# echo ${decrypted_share} - -# decrypted_share_base64=$(echo ${decrypted_share} | openssl base64 -A) - -# echo "${decrypted_share}" - - -# # Finally. submit encrypted share -# curl https://${node_rpc_address}/members/submitRecoveryShare ${@} -H "Content-Type: application/json" -d '{"recovery_share": "'${decrypted_share}'"}' - -# echo "" \ No newline at end of file +# Finally. submit encrypted share +curl https://${node_rpc_address}/members/submitRecoveryShare ${@} -H "Content-Type: application/json" -d '{"recovery_share": "'${decrypted_share}'"}' \ No newline at end of file From 5fc21829658a78393a99c8370cdcb60bd68c9006 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Wed, 27 May 2020 09:07:44 +0100 Subject: [PATCH 24/33] WIP --- tests/infra/member.py | 22 ++++++++-------------- tests/recovery.py | 11 ++++++----- tests/submit_recovery_share.sh | 2 +- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/tests/infra/member.py b/tests/infra/member.py index 3b360357d66f..275a88726b91 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -142,15 +142,17 @@ def ack(self, remote_node): def get_and_decrypt_recovery_share(self, remote_node, defunct_network_enc_pubk): with remote_node.member_client(member_id=self.member_id) as mc: - r = mc.rpc("getEncryptedRecoveryShare", params={}) + r = mc.get("getEncryptedRecoveryShare", params={}) ctx = infra.crypto.CryptoBoxCtx( - os.path.join(self.common_dir, f"member{self.member_id}_enc_priv.pem"), + os.path.join(self.common_dir, f"member{self.member_id}_enc_privk.pem"), defunct_network_enc_pubk, ) - nonce_bytes = bytes(r.result["nonce"]) - encrypted_share_bytes = bytes(r.result["encrypted_share"]) + nonce_bytes = base64.b64decode(r.result["nonce"]) + encrypted_share_bytes = base64.b64decode( + r.result["encrypted_recovery_share"] + ) return ctx.decrypt(encrypted_share_bytes, nonce_bytes) def get_and_submit_recovery_share(self, remote_node, defunct_network_enc_pubk): @@ -169,14 +171,6 @@ def get_and_submit_recovery_share(self, remote_node, defunct_network_enc_pubk): os.path.join(self.common_dir, f"member{self.member_id}_privk.pem"), "--cacert", os.path.join(self.common_dir, "networkcert.pem"), - "-i", + # "-i", + log_output=True, ).check_returncode() - - # def submit_recovery_share(self, remote_node, decrypted_recovery_share): - # with remote_node.member_client(member_id=self.member_id) as mc: - # r = mc.rpc( - # "submitRecoveryShare", - # params={"recovery_share": decrypted_recovery_share}, - # ) - # assert r.error is None, f"Error submitting recovery share: {r.error}" - # return r diff --git a/tests/recovery.py b/tests/recovery.py index f407a15a7e80..59c56e26a46d 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -52,15 +52,17 @@ def test_share_resilience(network, args): # TODO: Address issue of knowing when share is globally committed? check_commit = infra.checker.Checker(nc) - decrypted_share = m.get_and_decrypt_recovery_share( - primary, defunct_network_enc_pubk - ) + # decrypted_share = m.get_and_decrypt_recovery_share( + # primary, defunct_network_enc_pubk + # ) + m.get_and_submit_recovery_share(primary, defunct_network_enc_pubk) # check_commit( # m.submit_recovery_share(primary, decrypted_share), result=False # ) submitted_shares_count += 1 import time + time.sleep(2) # In theory, check_commit should be sufficient to guarantee that the new primary @@ -85,10 +87,9 @@ def test_share_resilience(network, args): new_primary is not primary ), f"Primary {primary.node_id} should have changed after election" - decrypted_share = last_member_to_submit.get_and_decrypt_recovery_share( + last_member_to_submit.get_and_submit_recovery_share( new_primary, defunct_network_enc_pubk ) - # last_member_to_submit.submit_recovery_share(new_primary, decrypted_share) for node in recovered_network.get_joined_nodes(): recovered_network.wait_for_state( diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh index 93f8459c2a6c..80ba0ec953de 100755 --- a/tests/submit_recovery_share.sh +++ b/tests/submit_recovery_share.sh @@ -64,7 +64,7 @@ fi # set -x # Retrieve encrypted recovery share and nonce -resp=$(curl -sS https://${node_rpc_address}/members/getEncryptedRecoveryShare ${@}) +resp=$(curl https://${node_rpc_address}/members/getEncryptedRecoveryShare ${@}) encrypted_share="$(echo ${resp} | jq -r .encrypted_recovery_share)" nonce="$(echo ${resp} | jq -r .nonce)" From 8a80de75b652adb48b66146c7b36303bfc40645e Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Mon, 15 Jun 2020 17:45:16 +0100 Subject: [PATCH 25/33] Cleanup and docs --- doc/members/accept_recovery.rst | 33 +++++++++--------------- src/node/rpc/member_frontend.h | 9 ++++--- src/utils/recovery_share_enc.cpp | 44 -------------------------------- tests/infra/member.py | 3 --- tests/memberclient.py | 2 +- tests/submit_recovery_share.sh | 14 +++++----- 6 files changed, 26 insertions(+), 79 deletions(-) delete mode 100644 src/utils/recovery_share_enc.cpp diff --git a/doc/members/accept_recovery.rst b/doc/members/accept_recovery.rst index 2aee743bd8c6..65401f881a17 100644 --- a/doc/members/accept_recovery.rst +++ b/doc/members/accept_recovery.rst @@ -42,28 +42,19 @@ To restore private transactions and complete the recovery procedure, members sho .. note:: The members who submit their recovery shares do not necessarily have to be the members who previously accepted the recovery. -First, members should retrieve their encrypted recovery shares via the ``recovery_share`` RPC [#recovery_share]_. +The recovery share retrieval, decryption and submission steps are conveniently performed by the ``submit_recovery_share.sh`` script as follows: .. code-block:: bash - $ curl https:///members/recovery_share -X GET --cacert network_cert --key member1_privk --cert member1_cert -H "content-type: application/json" + $ ./submit_recovery_share.sh --rpc-address --member-enc-privk member0_enc_privk.pem --network-enc-pubk network_enc_pubk --cert member0_cert + --key member0_privk --cacert network_cert + "1/2 recovery shares successfully submitted." -Then, members should decrypt their shares using their private encryption key and the `previous` network public encryption key (output by the first node of the now-defunct service via the ``network-enc-pubk-file`` :ref:`command line option `) using `NaCl's public-key authenticated encryption `_. + $ ./submit_recovery_share.sh --rpc-address --member-enc-privk member1_enc_privk.pem --network-enc-pubk network_enc_pubk --cert member1_cert + --key member1_privk --cacert network_cert + "2/2 recovery shares successfully submitted. End of recovery procedure initiated." -Finally, members should submit their decrypted share to CCF via the ``recovery_share/submit`` RPC: - -.. code-block:: bash - - $ cat submit_recovery_share.json - {"recovery_share": []} - - $ curl https:///members/recovery_share/submit -X POST --data-binary @submit_recovery_share.json --cacert network_cert --key member1_privk --cert member1_cert -H "content-type: application/json" - false - - $ curl https:///members/recovery_share/submit -X POST --data-binary @submit_recovery_share.json --cacert network_cert --key member2_privk --cert member2_cert -H "content-type: application/json" - true - -When the recovery threshold is reached, the ``recovery_share/submit`` RPC returns ``true``. At this point, the private recovery procedure is started and the private ledger is being recovered. +When the recovery threshold is reached, the ``recovery_share/submit`` RPC returns that the end of the recovery procedure is initiated and the private ledger is now being recovered. .. note:: While all nodes are recovering the private ledger, no new transaction can be executed by the network. @@ -91,16 +82,16 @@ Summary Diagram Node 2-->>Member 1: State: ACCEPTED Note over Node 2, Node 3: accept_recovery proposal completes. Service is ready to accept recovery shares. - Member 0->>+Node 2: recovery_share + Member 0->>+Node 2: GET recovery_share Node 2-->>Member 0: Encrypted recovery share for Member 0 Note over Member 0: Decrypts recovery share - Member 0->>+Node 2: recovery_share/submit: {"recovery_share": ...} + Member 0->>+Node 2: POST recovery_share/submit: {"recovery_share": ...} Node 2-->>Member 0: False - Member 1->>+Node 2: recovery_share + Member 1->>+Node 2: GET recovery_share Node 2-->>Member 1: Encrypted recovery share for Member 1 Note over Member 1: Decrypts recovery share - Member 1->>+Node 2: recovery_share/submit: {"recovery_share": ...} + Member 1->>+Node 2: POST recovery_share/submit: {"recovery_share": ...} Node 2-->>Member 1: True Note over Node 2, Node 3: Reading Private Ledger... diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index ae4d89fb127d..9e10e5f7d100 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -914,8 +914,7 @@ namespace ccf // The number of shares required to re-assemble the secret has not yet // been reached return make_success(fmt::format( - "Recovery share successfully submitted. {}/{} recovery shares " - "submitted.", + "{}/{} recovery shares successfully submitted.", submitted_shares_count, g.get_recovery_threshold())); } @@ -937,7 +936,11 @@ namespace ccf } share_manager.clear_submitted_recovery_shares(args.tx); - return make_success(true); + return make_success(fmt::format( + "{}/{} recovery shares successfully submitted. End of recovery " + "procedure initiated.", + submitted_shares_count, + g.get_recovery_threshold())); }; install( MemberProcs::SUBMIT_RECOVERY_SHARE, diff --git a/src/utils/recovery_share_enc.cpp b/src/utils/recovery_share_enc.cpp deleted file mode 100644 index 1233258d131d..000000000000 --- a/src/utils/recovery_share_enc.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the Apache 2.0 License. - -#include "ds/files.h" -#include "tls/base64.h" - -#include -#include - -int main(int argc, char** argv) -{ - CLI::App app{"recovery share enc"}; - - std::string member_privk_file; - app.add_option( - "--member-enc-privk-file", - member_privk_file, - "Member encryption private key file"); - - std::string network_pubk_file; - app.add_option( - "--network-enc-pubk-file", - network_pubk_file, - "Previous network encryption public key file"); - - std::string recovery_share; - app.add_option( - "--recovery_share", recovery_share, "Encrypted recovery share (base64)"); - - std::string nonce; - app.add_option("--nonce", nonce, "Nonce (base64)"); - - CLI11_PARSE(app, argc, argv); - - // TODO: - // 1. Build and install this in cmake - // 2. Crypto box open - // 3. Output base 64 encoded recovery share - - auto raw_recovery_share = tls::raw_from_b64(recovery_share); - auto raw_nonce = tls::raw_from_b64(nonce); - - -} \ No newline at end of file diff --git a/tests/infra/member.py b/tests/infra/member.py index 90b233887b66..d9ad1dddf4c3 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -166,7 +166,6 @@ def get_and_decrypt_recovery_share(self, remote_node, defunct_network_enc_pubk): def get_and_submit_recovery_share(self, remote_node, defunct_network_enc_pubk): # For now, all members are given an encryption key (for recovery) - input("") infra.proc.ccall( self.share_script, "--rpc-address", @@ -181,7 +180,5 @@ def get_and_submit_recovery_share(self, remote_node, defunct_network_enc_pubk): os.path.join(self.common_dir, f"member{self.member_id}_privk.pem"), "--cacert", os.path.join(self.common_dir, "networkcert.pem"), - "-i", - "-vv", log_output=True, ).check_returncode() diff --git a/tests/memberclient.py b/tests/memberclient.py index 60d5e07e8f39..26e71bdded39 100644 --- a/tests/memberclient.py +++ b/tests/memberclient.py @@ -42,7 +42,7 @@ def test_add_member(network, args): ) try: - new_member.get_and_submit_recovery_share( + new_member.get_and_decrypt_recovery_share( primary, network.store_current_network_encryption_key() ) assert False, "New accepted members are not given recovery shares" diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh index 1986b5c66cfa..092aa3c6a299 100755 --- a/tests/submit_recovery_share.sh +++ b/tests/submit_recovery_share.sh @@ -63,26 +63,26 @@ if [ -z "$network_enc_pubk" ]; then fi # Retrieve encrypted recovery share and nonce -resp=$(curl -X GET https://${node_rpc_address}/members/recovery_share ${@}) -encrypted_share="$(echo ${resp} | jq -r .encrypted_recovery_share)" -nonce="$(echo ${resp} | jq -r .nonce)" +resp=$(curl --fail -sS -X GET https://"${node_rpc_address}"/members/recovery_share "${@}") +encrypted_share="$(echo "${resp}" | jq -r .encrypted_recovery_share)" +nonce="$(echo "${resp}" | jq -r .nonce)" echo "Encrypted recovery share: ${encrypted_share}" echo "Nonce: ${nonce}" # Parse raw private key from SubjectPublicKeyInfo DER format, as generated by keygenerator.sh --gen-enc-key der_header_privk_len=14 -openssl asn1parse -in ${member_enc_privk} -strparse ${der_header_privk_len} -out member_enc_privk.raw -noout +openssl asn1parse -in "${member_enc_privk}" -strparse "${der_header_privk_len}" -out member_enc_privk.raw -noout # Parse raw public key generated by network der_header_pubk_len=9 -openssl asn1parse -in ${network_enc_pubk} -i -strparse ${der_header_pubk_len} -out network_enc_pubk.raw -noout +openssl asn1parse -in "${network_enc_pubk}" -i -strparse "${der_header_pubk_len}" -out network_enc_pubk.raw -noout # step CLI base64 standard to URL encoding -encrypted_share_b64_url=$(echo ${encrypted_share} | sed 's/+/-/g; s/\//_/g') +encrypted_share_b64_url=$(echo "${encrypted_share}" | sed 's/+/-/g; s/\//_/g') # Decrypt encrypted share with nonce, member private key and previous network public key decrypted_share=$(echo "${encrypted_share_b64_url}" | step crypto nacl box open base64:"${nonce}" network_enc_pubk.raw member_enc_privk.raw | openssl base64 -A) # Finally. submit encrypted share -curl https://${node_rpc_address}/members/recovery_share/submit ${@} -H "Content-Type: application/json" -d '{"recovery_share": "'${decrypted_share}'"}' \ No newline at end of file +curl -sS --fail https://"${node_rpc_address}"/members/recovery_share/submit "${@}" -H "Content-Type: application/json" -d '{"recovery_share": "'"${decrypted_share}"'"}' \ No newline at end of file From 41ea01ae2446b0f848dfea8f535f110151585217 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Mon, 15 Jun 2020 17:49:55 +0100 Subject: [PATCH 26/33] Cleanup --- src/node/node_state.h | 5 +-- start_test_network.sh | 16 ++++----- tests/code_update.py | 84 +++++++++++++++++++++---------------------- tests/memberclient.py | 1 - 4 files changed, 53 insertions(+), 53 deletions(-) diff --git a/src/node/node_state.h b/src/node/node_state.h index 3573c8cc1d75..6f466132b43b 100644 --- a/src/node/node_state.h +++ b/src/node/node_state.h @@ -938,8 +938,9 @@ namespace ccf bool is_primary() const override { return ( - sm.check(State::partOfNetwork) && - sm.check(State::partOfPublicNetwork) && consensus->is_primary()); + (sm.check(State::partOfNetwork) || + sm.check(State::partOfPublicNetwork)) && + consensus->is_primary()); } bool is_part_of_network() const override diff --git a/start_test_network.sh b/start_test_network.sh index bf39b7592ca8..a09425cfe828 100755 --- a/start_test_network.sh +++ b/start_test_network.sh @@ -4,17 +4,17 @@ set -e -# echo "Setting up Python environment..." -# if [ ! -f "env/bin/activate" ] -# then -# python3.7 -m venv env -# fi -# source env/bin/activate +echo "Setting up Python environment..." +if [ ! -f "env/bin/activate" ] + then + python3.7 -m venv env +fi +source env/bin/activate PATH_HERE=$(dirname "$(realpath -s "$0")") -# pip install -q -U -r "${PATH_HERE}"/tests/requirements.txt -# echo "Python environment successfully setup" +pip install -q -U -r "${PATH_HERE}"/tests/requirements.txt +echo "Python environment successfully setup" CURL_CLIENT=ON \ python "${PATH_HERE}"/tests/start_network.py \ diff --git a/tests/code_update.py b/tests/code_update.py index cb2a85c1ccb1..824a1e165539 100644 --- a/tests/code_update.py +++ b/tests/code_update.py @@ -54,48 +54,48 @@ def run(args): code_not_found_exception is not None ), f"Adding a node with unsupported code id {new_code_id} should fail" - # # Slow quote verification means that any attempt to add a node may cause an election, so confirm primary after adding node - # primary, _ = network.find_primary() - - # network.consortium.add_new_code(primary, new_code_id) - - # new_nodes = set() - # old_nodes_count = len(network.nodes) - # new_nodes_count = old_nodes_count + 1 - - # LOG.info( - # f"Adding more new nodes ({new_nodes_count}) than originally existed ({old_nodes_count})" - # ) - # for _ in range(0, new_nodes_count): - # new_node = network.create_and_trust_node( - # args.patched_file_name, "localhost", args - # ) - # assert new_node - # new_nodes.add(new_node) - - # LOG.info("Stopping all original nodes") - # old_nodes = set(network.nodes).difference(new_nodes) - # for node in old_nodes: - # LOG.debug(f"Stopping old node {node.node_id}") - # node.stop() - - # sleep_time = ( - # args.pbft_view_change_timeout * 2 / 1000 - # if args.consensus == "pbft" - # else args.raft_election_timeout * 2 / 1000 - # ) - # LOG.info(f"Waiting {sleep_time}s for a new primary to be elected...") - # time.sleep(sleep_time) - - # new_primary, _ = network.find_primary() - # LOG.info(f"Waited, new_primary is {new_primary.node_id}") - - # LOG.info("Adding another node to the network") - # new_node = network.create_and_trust_node( - # args.patched_file_name, "localhost", args - # ) - # assert new_node - # network.wait_for_node_commit_sync(args.consensus) + # Slow quote verification means that any attempt to add a node may cause an election, so confirm primary after adding node + primary, _ = network.find_primary() + + network.consortium.add_new_code(primary, new_code_id) + + new_nodes = set() + old_nodes_count = len(network.nodes) + new_nodes_count = old_nodes_count + 1 + + LOG.info( + f"Adding more new nodes ({new_nodes_count}) than originally existed ({old_nodes_count})" + ) + for _ in range(0, new_nodes_count): + new_node = network.create_and_trust_node( + args.patched_file_name, "localhost", args + ) + assert new_node + new_nodes.add(new_node) + + LOG.info("Stopping all original nodes") + old_nodes = set(network.nodes).difference(new_nodes) + for node in old_nodes: + LOG.debug(f"Stopping old node {node.node_id}") + node.stop() + + sleep_time = ( + args.pbft_view_change_timeout * 2 / 1000 + if args.consensus == "pbft" + else args.raft_election_timeout * 2 / 1000 + ) + LOG.info(f"Waiting {sleep_time}s for a new primary to be elected...") + time.sleep(sleep_time) + + new_primary, _ = network.find_primary() + LOG.info(f"Waited, new_primary is {new_primary.node_id}") + + LOG.info("Adding another node to the network") + new_node = network.create_and_trust_node( + args.patched_file_name, "localhost", args + ) + assert new_node + network.wait_for_node_commit_sync(args.consensus) if __name__ == "__main__": diff --git a/tests/memberclient.py b/tests/memberclient.py index 26e71bdded39..c04414f5e7da 100644 --- a/tests/memberclient.py +++ b/tests/memberclient.py @@ -81,7 +81,6 @@ def assert_recovery_shares_update(func, network, args, **kwargs): network.store_current_network_encryption_key() already_active_member = network.consortium.get_any_active_member() defunct_network_enc_pubk = network.store_current_network_encryption_key() - # TODO: Only get the encrypted share saved_share = already_active_member.get_and_decrypt_recovery_share( primary, defunct_network_enc_pubk ) From 515869c72dbedf250b30b44e7419c9ea030ef26c Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 16 Jun 2020 10:21:24 +0100 Subject: [PATCH 27/33] Check for commit of submitted share --- doc/members/accept_recovery.rst | 12 ++++++++++++ tests/infra/checker.py | 32 ++++++++++++++++---------------- tests/infra/consortium.py | 13 +++++-------- tests/infra/member.py | 8 ++++---- tests/recovery.py | 15 +++------------ tests/submit_recovery_share.sh | 13 +++++-------- 6 files changed, 45 insertions(+), 48 deletions(-) diff --git a/doc/members/accept_recovery.rst b/doc/members/accept_recovery.rst index 65401f881a17..6fe55d28b34d 100644 --- a/doc/members/accept_recovery.rst +++ b/doc/members/accept_recovery.rst @@ -48,10 +48,22 @@ The recovery share retrieval, decryption and submission steps are conveniently p $ ./submit_recovery_share.sh --rpc-address --member-enc-privk member0_enc_privk.pem --network-enc-pubk network_enc_pubk --cert member0_cert --key member0_privk --cacert network_cert + HTTP/1.1 200 OK + content-length: 46 + content-type: application/json + x-ccf-global-commit: 27 + x-ccf-tx-seqno: 28 + x-ccf-tx-view: 4 "1/2 recovery shares successfully submitted." $ ./submit_recovery_share.sh --rpc-address --member-enc-privk member1_enc_privk.pem --network-enc-pubk network_enc_pubk --cert member1_cert --key member1_privk --cacert network_cert + HTTP/1.1 200 OK + content-length: 83 + content-type: application/json + x-ccf-global-commit: 29 + x-ccf-tx-seqno: 30 + x-ccf-tx-view: 4 "2/2 recovery shares successfully submitted. End of recovery procedure initiated." When the recovery threshold is reached, the ``recovery_share/submit`` RPC returns that the end of the recovery procedure is initiated and the private ledger is now being recovered. diff --git a/tests/infra/checker.py b/tests/infra/checker.py index 6a658a586c27..47ff0bba6c41 100644 --- a/tests/infra/checker.py +++ b/tests/infra/checker.py @@ -71,20 +71,20 @@ def __call__(self, rpc_result, result=None, error=None, timeout=2): assert rpc_result.seqno and rpc_result.view, rpc_result - if self.client: - wait_for_global_commit(self.client, rpc_result.seqno, rpc_result.view) + if self.client: + wait_for_global_commit(self.client, rpc_result.seqno, rpc_result.view) - if self.notification_queue: - end_time = time.time() + timeout - while time.time() < end_time: - while self.notification_queue.not_empty: - notification = self.notification_queue.get() - n = json.loads(notification)["commit"] - assert ( - n > self.notified_commit - ), f"Received notification of commit {n} after commit {self.notified_commit}" - self.notified_commit = n - if n >= rpc_result.seqno: - return - time.sleep(0.5) - raise TimeoutError("Timed out waiting for notification") + if self.notification_queue: + end_time = time.time() + timeout + while time.time() < end_time: + while self.notification_queue.not_empty: + notification = self.notification_queue.get() + n = json.loads(notification)["commit"] + assert ( + n > self.notified_commit + ), f"Received notification of commit {n} after commit {self.notified_commit}" + self.notified_commit = n + if n >= rpc_result.seqno: + return + time.sleep(0.5) + raise TimeoutError("Timed out waiting for notification") diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 565a2d7505c3..3518cd5ca0ac 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -330,18 +330,15 @@ def recover_with_shares(self, remote_node, defunct_network_enc_pubk): check_commit = infra.checker.Checker(nc) for m in self.get_active_members(): - m.get_and_submit_recovery_share(remote_node, defunct_network_enc_pubk) - # r = m.submit_recovery_share(remote_node, decrypted_share) + r = m.get_and_submit_recovery_share(remote_node, defunct_network_enc_pubk) submitted_shares_count += 1 - # check_commit( - # r, - # result=True - # if submitted_shares_count >= self.recovery_threshold - # else False, - # ) + check_commit(r) if submitted_shares_count >= self.recovery_threshold: + assert "End of recovery procedure initiated" in r.result break + else: + assert "End of recovery procedure initiated" not in r.result def set_recovery_threshold(self, remote_node, recovery_threshold): script = """ diff --git a/tests/infra/member.py b/tests/infra/member.py index d9ad1dddf4c3..4c943a201d41 100644 --- a/tests/infra/member.py +++ b/tests/infra/member.py @@ -10,8 +10,6 @@ import os import base64 -from loguru import logger as LOG - class NoRecoveryShareFound(Exception): def __init__(self, response): @@ -166,7 +164,7 @@ def get_and_decrypt_recovery_share(self, remote_node, defunct_network_enc_pubk): def get_and_submit_recovery_share(self, remote_node, defunct_network_enc_pubk): # For now, all members are given an encryption key (for recovery) - infra.proc.ccall( + res = infra.proc.ccall( self.share_script, "--rpc-address", f"{remote_node.host}:{remote_node.rpc_port}", @@ -181,4 +179,6 @@ def get_and_submit_recovery_share(self, remote_node, defunct_network_enc_pubk): "--cacert", os.path.join(self.common_dir, "networkcert.pem"), log_output=True, - ).check_returncode() + ) + res.check_returncode() + return infra.clients.Response.from_raw(res.stdout) diff --git a/tests/recovery.py b/tests/recovery.py index 59c56e26a46d..87cfb45bd761 100644 --- a/tests/recovery.py +++ b/tests/recovery.py @@ -50,21 +50,12 @@ def test_share_resilience(network, args): last_member_to_submit = m break - # TODO: Address issue of knowing when share is globally committed? check_commit = infra.checker.Checker(nc) - # decrypted_share = m.get_and_decrypt_recovery_share( - # primary, defunct_network_enc_pubk - # ) - m.get_and_submit_recovery_share(primary, defunct_network_enc_pubk) - # check_commit( - # m.submit_recovery_share(primary, decrypted_share), result=False - # ) + check_commit( + m.get_and_submit_recovery_share(primary, defunct_network_enc_pubk) + ) submitted_shares_count += 1 - import time - - time.sleep(2) - # In theory, check_commit should be sufficient to guarantee that the new primary # will know about all the recovery shares submitted so far. However, because of # https://github.com/microsoft/CCF/issues/589, we have to wait for all nodes diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh index 092aa3c6a299..73900aaf18dd 100755 --- a/tests/submit_recovery_share.sh +++ b/tests/submit_recovery_share.sh @@ -63,13 +63,10 @@ if [ -z "$network_enc_pubk" ]; then fi # Retrieve encrypted recovery share and nonce -resp=$(curl --fail -sS -X GET https://"${node_rpc_address}"/members/recovery_share "${@}") +resp=$(curl -sS --fail -X GET https://"${node_rpc_address}"/members/recovery_share "${@}") encrypted_share="$(echo "${resp}" | jq -r .encrypted_recovery_share)" nonce="$(echo "${resp}" | jq -r .nonce)" -echo "Encrypted recovery share: ${encrypted_share}" -echo "Nonce: ${nonce}" - # Parse raw private key from SubjectPublicKeyInfo DER format, as generated by keygenerator.sh --gen-enc-key der_header_privk_len=14 openssl asn1parse -in "${member_enc_privk}" -strparse "${der_header_privk_len}" -out member_enc_privk.raw -noout @@ -78,11 +75,11 @@ openssl asn1parse -in "${member_enc_privk}" -strparse "${der_header_privk_len}" der_header_pubk_len=9 openssl asn1parse -in "${network_enc_pubk}" -i -strparse "${der_header_pubk_len}" -out network_enc_pubk.raw -noout -# step CLI base64 standard to URL encoding +# step CLI base64 standard to URL encoding for input encrypted_share_b64_url=$(echo "${encrypted_share}" | sed 's/+/-/g; s/\//_/g') -# Decrypt encrypted share with nonce, member private key and previous network public key +# Decrypt encrypted share with nonce, member private key and defunct network public key decrypted_share=$(echo "${encrypted_share_b64_url}" | step crypto nacl box open base64:"${nonce}" network_enc_pubk.raw member_enc_privk.raw | openssl base64 -A) -# Finally. submit encrypted share -curl -sS --fail https://"${node_rpc_address}"/members/recovery_share/submit "${@}" -H "Content-Type: application/json" -d '{"recovery_share": "'"${decrypted_share}"'"}' \ No newline at end of file +# Finally, submit decrypted share +curl -i -sS --fail https://"${node_rpc_address}"/members/recovery_share/submit "${@}" -H "Content-Type: application/json" -d '{"recovery_share": "'"${decrypted_share}"'"}' \ No newline at end of file From 95b393b0d228b4620c1d9daf130d1d78c8a2be1b Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 16 Jun 2020 10:57:39 +0100 Subject: [PATCH 28/33] Schema, fix build --- doc/schemas/recovery_share/submit_params.json | 7 +--- doc/schemas/recovery_share/submit_result.json | 2 +- doc/schemas/recovery_share_result.json | 20 +++------- src/node/rpc/test/member_voting_test.cpp | 40 ++++++++++--------- tests/infra/consortium.py | 4 +- 5 files changed, 31 insertions(+), 42 deletions(-) diff --git a/doc/schemas/recovery_share/submit_params.json b/doc/schemas/recovery_share/submit_params.json index e1cb5fd879a5..e312316f10ea 100644 --- a/doc/schemas/recovery_share/submit_params.json +++ b/doc/schemas/recovery_share/submit_params.json @@ -2,12 +2,7 @@ "$schema": "http://json-schema.org/draft-07/schema#", "properties": { "recovery_share": { - "items": { - "maximum": 255, - "minimum": 0, - "type": "integer" - }, - "type": "array" + "type": "string" } }, "required": [ diff --git a/doc/schemas/recovery_share/submit_result.json b/doc/schemas/recovery_share/submit_result.json index 6af279ebee53..19668eb75026 100644 --- a/doc/schemas/recovery_share/submit_result.json +++ b/doc/schemas/recovery_share/submit_result.json @@ -1,5 +1,5 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "title": "recovery_share/submit/result", - "type": "boolean" + "type": "string" } \ No newline at end of file diff --git a/doc/schemas/recovery_share_result.json b/doc/schemas/recovery_share_result.json index 8d0c78d35af2..4b59a933ecff 100644 --- a/doc/schemas/recovery_share_result.json +++ b/doc/schemas/recovery_share_result.json @@ -1,26 +1,16 @@ { "$schema": "http://json-schema.org/draft-07/schema#", "properties": { - "encrypted_share": { - "items": { - "maximum": 255, - "minimum": 0, - "type": "integer" - }, - "type": "array" + "encrypted_recovery_share": { + "type": "string" }, "nonce": { - "items": { - "maximum": 255, - "minimum": 0, - "type": "integer" - }, - "type": "array" + "type": "string" } }, "required": [ - "nonce", - "encrypted_share" + "encrypted_recovery_share", + "nonce" ], "title": "recovery_share/result", "type": "object" diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 4a639e579806..0fd47a622cec 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -1643,17 +1643,20 @@ DOCTEST_TEST_CASE("Submit recovery shares") DOCTEST_INFO("Retrieve and decrypt recovery shares"); { - const auto get_recovery_shares = - create_request(nullptr, "recovery_share", HTTP_GET); + const auto get_recovery_shares = create_request( + nullptr, MemberProcs::GET_ENCRYPTED_RECOVERY_SHARE, HTTP_GET); for (auto const& m : members) { - auto encrypted_share = parse_response_body( + auto resp = parse_response_body( frontend_process(frontend, get_recovery_shares, m.second.first)); + auto encrypted_share = tls::raw_from_b64(resp.encrypted_recovery_share); + auto nonce = tls::raw_from_b64(resp.nonce); + retrieved_shares[m.first] = crypto::Box::open( - encrypted_share.encrypted_share, - encrypted_share.nonce, + encrypted_share, + nonce, crypto::BoxKey::public_from_private( network.encryption_key->private_raw), m.second.second); @@ -1664,8 +1667,8 @@ DOCTEST_TEST_CASE("Submit recovery shares") { MemberId member_id = 0; const auto submit_recovery_share = create_request( - SubmitRecoveryShare({retrieved_shares[member_id]}), - "recovery_share/submit"); + SubmitRecoveryShare({tls::b64_from_raw(retrieved_shares[member_id])}), + MemberProcs::SUBMIT_RECOVERY_SHARE); check_error( frontend_process( @@ -1697,7 +1700,8 @@ DOCTEST_TEST_CASE("Submit recovery shares") auto bogus_recovery_share = retrieved_shares[m.first]; bogus_recovery_share[0] = bogus_recovery_share[0] + 1; const auto submit_recovery_share = create_request( - SubmitRecoveryShare({bogus_recovery_share}), "recovery_share/submit"); + SubmitRecoveryShare({tls::b64_from_raw(bogus_recovery_share)}), + MemberProcs::SUBMIT_RECOVERY_SHARE); auto rep = frontend_process(frontend, submit_recovery_share, m.second.first); @@ -1706,11 +1710,7 @@ DOCTEST_TEST_CASE("Submit recovery shares") // Share submission should only complete when the recovery threshold // has been reached - if (submitted_shares_count < recovery_threshold) - { - DOCTEST_REQUIRE(parse_response_body(rep) == false); - } - else + if (submitted_shares_count >= recovery_threshold) { check_error(rep, HTTP_STATUS_INTERNAL_SERVER_ERROR); break; @@ -1727,19 +1727,21 @@ DOCTEST_TEST_CASE("Submit recovery shares") for (auto const& m : members) { const auto submit_recovery_share = create_request( - SubmitRecoveryShare({retrieved_shares[m.first]}), - "recovery_share/submit"); + SubmitRecoveryShare({tls::b64_from_raw(retrieved_shares[m.first])}), + MemberProcs::SUBMIT_RECOVERY_SHARE); - auto ret = parse_response_body( - frontend_process(frontend, submit_recovery_share, m.second.first)); + auto rep = + frontend_process(frontend, submit_recovery_share, m.second.first); submitted_shares_count++; // Share submission should only complete when the recovery threshold // has been reached - DOCTEST_REQUIRE(ret == (submitted_shares_count >= recovery_threshold)); - if (ret) + if (submitted_shares_count >= recovery_threshold) { + DOCTEST_REQUIRE( + parse_response_body(rep).find( + "End of recovery procedure initiated.") != std::string::npos); break; } } diff --git a/tests/infra/consortium.py b/tests/infra/consortium.py index 3518cd5ca0ac..1abfa03f87bd 100644 --- a/tests/infra/consortium.py +++ b/tests/infra/consortium.py @@ -330,7 +330,9 @@ def recover_with_shares(self, remote_node, defunct_network_enc_pubk): check_commit = infra.checker.Checker(nc) for m in self.get_active_members(): - r = m.get_and_submit_recovery_share(remote_node, defunct_network_enc_pubk) + r = m.get_and_submit_recovery_share( + remote_node, defunct_network_enc_pubk + ) submitted_shares_count += 1 check_commit(r) From e6361e00cd98ba36c0c7e7cc0d4d72cddc99d510 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 16 Jun 2020 12:21:14 +0100 Subject: [PATCH 29/33] Submit share from JSON to Text --- src/node/rpc/member_frontend.h | 56 ++++++++++++------------ src/node/rpc/test/member_voting_test.cpp | 30 ++++++++++--- tests/submit_recovery_share.sh | 12 ++--- 3 files changed, 58 insertions(+), 40 deletions(-) diff --git a/src/node/rpc/member_frontend.h b/src/node/rpc/member_frontend.h index 9e10e5f7d100..e59c1fa60e24 100644 --- a/src/node/rpc/member_frontend.h +++ b/src/node/rpc/member_frontend.h @@ -30,13 +30,6 @@ namespace ccf DECLARE_JSON_REQUIRED_FIELDS(SetUserData, user_id) DECLARE_JSON_OPTIONAL_FIELDS(SetUserData, user_data) - struct SubmitRecoveryShare - { - std::string recovery_share; - }; - DECLARE_JSON_TYPE(SubmitRecoveryShare) - DECLARE_JSON_REQUIRED_FIELDS(SubmitRecoveryShare, recovery_share) - struct GetEncryptedRecoveryShare { std::string encrypted_recovery_share; @@ -869,32 +862,36 @@ namespace ccf .set_auto_schema() .set_http_get_only(); - auto submit_recovery_share = [this]( - RequestArgs& args, - nlohmann::json&& params) { + auto submit_recovery_share = [this](ccf::RequestArgs& args) { // Only active members can submit their shares for recovery if (!check_member_active(args.tx, args.caller_id)) { - return make_error(HTTP_STATUS_FORBIDDEN, "Member is not active"); + args.rpc_ctx->set_response_status(HTTP_STATUS_FORBIDDEN); + args.rpc_ctx->set_response_body("Member is not active"); + return; } GenesisGenerator g(this->network, args.tx); if ( g.get_service_status() != ServiceStatus::WAITING_FOR_RECOVERY_SHARES) { - return make_error( - HTTP_STATUS_FORBIDDEN, + args.rpc_ctx->set_response_status(HTTP_STATUS_FORBIDDEN); + args.rpc_ctx->set_response_body( "Service is not waiting for recovery shares"); + return; } if (node.is_reading_private_ledger()) { - return make_error( - HTTP_STATUS_FORBIDDEN, "Node is already recovering private ledger"); + args.rpc_ctx->set_response_status(HTTP_STATUS_FORBIDDEN); + args.rpc_ctx->set_response_body( + "Node is already recovering private ledger"); + return; } - const auto in = params.get(); - auto raw_recovery_share = tls::raw_from_b64(in.recovery_share); + const auto& in = args.rpc_ctx->get_request_body(); + const auto s = std::string(in.begin(), in.end()); + auto raw_recovery_share = tls::raw_from_b64(s); size_t submitted_shares_count = 0; try @@ -904,19 +901,22 @@ namespace ccf } catch (const std::logic_error& e) { - return make_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, + args.rpc_ctx->set_response_status(HTTP_STATUS_INTERNAL_SERVER_ERROR); + args.rpc_ctx->set_response_body( fmt::format("Could not submit recovery share: {}", e.what())); + return; } if (submitted_shares_count < g.get_recovery_threshold()) { // The number of shares required to re-assemble the secret has not yet // been reached - return make_success(fmt::format( + args.rpc_ctx->set_response_status(HTTP_STATUS_OK); + args.rpc_ctx->set_response_body(fmt::format( "{}/{} recovery shares successfully submitted.", submitted_shares_count, g.get_recovery_threshold())); + return; } LOG_DEBUG_FMT( @@ -930,23 +930,23 @@ namespace ccf { // For now, clear the submitted shares if combination fails. share_manager.clear_submitted_recovery_shares(args.tx); - return make_error( - HTTP_STATUS_INTERNAL_SERVER_ERROR, + args.rpc_ctx->set_response_status(HTTP_STATUS_INTERNAL_SERVER_ERROR); + args.rpc_ctx->set_response_body( fmt::format("Failed to initiate private recovery: {}", e.what())); + return; } share_manager.clear_submitted_recovery_shares(args.tx); - return make_success(fmt::format( + + args.rpc_ctx->set_response_status(HTTP_STATUS_OK); + args.rpc_ctx->set_response_body(fmt::format( "{}/{} recovery shares successfully submitted. End of recovery " "procedure initiated.", submitted_shares_count, g.get_recovery_threshold())); }; - install( - MemberProcs::SUBMIT_RECOVERY_SHARE, - json_adapter(submit_recovery_share), - Write) - .set_auto_schema(); + install(MemberProcs::SUBMIT_RECOVERY_SHARE, submit_recovery_share, Write) + .set_auto_schema(); auto create = [this](kv::Tx& tx, nlohmann::json&& params) { LOG_DEBUG_FMT("Processing create RPC"); diff --git a/src/node/rpc/test/member_voting_test.cpp b/src/node/rpc/test/member_voting_test.cpp index 0fd47a622cec..d4b1d0ca91db 100644 --- a/src/node/rpc/test/member_voting_test.cpp +++ b/src/node/rpc/test/member_voting_test.cpp @@ -74,6 +74,11 @@ T parse_response_body(const TResponse& r) return body_j.get(); } +std::string parse_response_body(const TResponse& r) +{ + return std::string(r.body.begin(), r.body.end()); +} + void check_error(const TResponse& r, http_status expected) { DOCTEST_CHECK(r.status == expected); @@ -92,6 +97,17 @@ void set_whitelists(GenesisGenerator& gen) gen.set_whitelist(wl.first, wl.second); } +std::vector create_text_request( + const std::string& text, + const string& method_name, + http_method verb = HTTP_POST) +{ + http::Request r(method_name, verb); + const auto body = std::vector(text.begin(), text.end()); + r.set_body(&body); + return r.build_request(); +} + std::vector create_request( const json& params, const string& method_name, http_method verb = HTTP_POST) { @@ -1666,8 +1682,8 @@ DOCTEST_TEST_CASE("Submit recovery shares") DOCTEST_INFO("Submit share before the service is in correct state"); { MemberId member_id = 0; - const auto submit_recovery_share = create_request( - SubmitRecoveryShare({tls::b64_from_raw(retrieved_shares[member_id])}), + const auto submit_recovery_share = create_text_request( + tls::b64_from_raw(retrieved_shares[member_id]), MemberProcs::SUBMIT_RECOVERY_SHARE); check_error( @@ -1699,8 +1715,8 @@ DOCTEST_TEST_CASE("Submit recovery shares") { auto bogus_recovery_share = retrieved_shares[m.first]; bogus_recovery_share[0] = bogus_recovery_share[0] + 1; - const auto submit_recovery_share = create_request( - SubmitRecoveryShare({tls::b64_from_raw(bogus_recovery_share)}), + const auto submit_recovery_share = create_text_request( + tls::b64_from_raw(bogus_recovery_share), MemberProcs::SUBMIT_RECOVERY_SHARE); auto rep = @@ -1726,8 +1742,8 @@ DOCTEST_TEST_CASE("Submit recovery shares") size_t submitted_shares_count = 0; for (auto const& m : members) { - const auto submit_recovery_share = create_request( - SubmitRecoveryShare({tls::b64_from_raw(retrieved_shares[m.first])}), + const auto submit_recovery_share = create_text_request( + tls::b64_from_raw(retrieved_shares[m.first]), MemberProcs::SUBMIT_RECOVERY_SHARE); auto rep = @@ -1740,7 +1756,7 @@ DOCTEST_TEST_CASE("Submit recovery shares") if (submitted_shares_count >= recovery_threshold) { DOCTEST_REQUIRE( - parse_response_body(rep).find( + parse_response_body(rep).find( "End of recovery procedure initiated.") != std::string::npos); break; } diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh index 73900aaf18dd..ab9437318aa1 100755 --- a/tests/submit_recovery_share.sh +++ b/tests/submit_recovery_share.sh @@ -67,6 +67,10 @@ resp=$(curl -sS --fail -X GET https://"${node_rpc_address}"/members/recovery_sha encrypted_share="$(echo "${resp}" | jq -r .encrypted_recovery_share)" nonce="$(echo "${resp}" | jq -r .nonce)" + +## TODO: Mkstemp + + # Parse raw private key from SubjectPublicKeyInfo DER format, as generated by keygenerator.sh --gen-enc-key der_header_privk_len=14 openssl asn1parse -in "${member_enc_privk}" -strparse "${der_header_privk_len}" -out member_enc_privk.raw -noout @@ -78,8 +82,6 @@ openssl asn1parse -in "${network_enc_pubk}" -i -strparse "${der_header_pubk_len} # step CLI base64 standard to URL encoding for input encrypted_share_b64_url=$(echo "${encrypted_share}" | sed 's/+/-/g; s/\//_/g') -# Decrypt encrypted share with nonce, member private key and defunct network public key -decrypted_share=$(echo "${encrypted_share_b64_url}" | step crypto nacl box open base64:"${nonce}" network_enc_pubk.raw member_enc_privk.raw | openssl base64 -A) - -# Finally, submit decrypted share -curl -i -sS --fail https://"${node_rpc_address}"/members/recovery_share/submit "${@}" -H "Content-Type: application/json" -d '{"recovery_share": "'"${decrypted_share}"'"}' \ No newline at end of file +# Decrypt encrypted share with nonce, member private key and defunct network public key and submit share +# All in one line so that the recovery share is not exposed +echo "${encrypted_share_b64_url}" | step crypto nacl box open base64:"${nonce}" network_enc_pubk.raw member_enc_privk.raw | openssl base64 -A | curl -i -sS --fail https://"${node_rpc_address}"/members/recovery_share/submit "${@}" -d @- \ No newline at end of file From defc04e99f5978d277f4875551fee5d61e1a2ccb Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 16 Jun 2020 12:21:39 +0100 Subject: [PATCH 30/33] Remove global commit --- doc/members/accept_recovery.rst | 2 -- 1 file changed, 2 deletions(-) diff --git a/doc/members/accept_recovery.rst b/doc/members/accept_recovery.rst index 6fe55d28b34d..8feaef673830 100644 --- a/doc/members/accept_recovery.rst +++ b/doc/members/accept_recovery.rst @@ -51,7 +51,6 @@ The recovery share retrieval, decryption and submission steps are conveniently p HTTP/1.1 200 OK content-length: 46 content-type: application/json - x-ccf-global-commit: 27 x-ccf-tx-seqno: 28 x-ccf-tx-view: 4 "1/2 recovery shares successfully submitted." @@ -61,7 +60,6 @@ The recovery share retrieval, decryption and submission steps are conveniently p HTTP/1.1 200 OK content-length: 83 content-type: application/json - x-ccf-global-commit: 29 x-ccf-tx-seqno: 30 x-ccf-tx-view: 4 "2/2 recovery shares successfully submitted. End of recovery procedure initiated." From 3a2adbdea81de959a379aa9553198f610b74b49e Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 16 Jun 2020 12:23:01 +0100 Subject: [PATCH 31/33] Cleanup docs --- doc/members/accept_recovery.rst | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/doc/members/accept_recovery.rst b/doc/members/accept_recovery.rst index 8feaef673830..ae6991a3ace9 100644 --- a/doc/members/accept_recovery.rst +++ b/doc/members/accept_recovery.rst @@ -49,20 +49,18 @@ The recovery share retrieval, decryption and submission steps are conveniently p $ ./submit_recovery_share.sh --rpc-address --member-enc-privk member0_enc_privk.pem --network-enc-pubk network_enc_pubk --cert member0_cert --key member0_privk --cacert network_cert HTTP/1.1 200 OK - content-length: 46 - content-type: application/json + content-type: text/plain x-ccf-tx-seqno: 28 x-ccf-tx-view: 4 - "1/2 recovery shares successfully submitted." + 1/2 recovery shares successfully submitted. $ ./submit_recovery_share.sh --rpc-address --member-enc-privk member1_enc_privk.pem --network-enc-pubk network_enc_pubk --cert member1_cert --key member1_privk --cacert network_cert HTTP/1.1 200 OK - content-length: 83 - content-type: application/json + content-type: text/plain x-ccf-tx-seqno: 30 x-ccf-tx-view: 4 - "2/2 recovery shares successfully submitted. End of recovery procedure initiated." + 2/2 recovery shares successfully submitted. End of recovery procedure initiated. When the recovery threshold is reached, the ``recovery_share/submit`` RPC returns that the end of the recovery procedure is initiated and the private ledger is now being recovered. From 91a45b968867b071d2bb97ab3f7fd9073d9ccb47 Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 16 Jun 2020 12:32:01 +0100 Subject: [PATCH 32/33] mktemp --- tests/submit_recovery_share.sh | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/submit_recovery_share.sh b/tests/submit_recovery_share.sh index ab9437318aa1..2494104a48f4 100755 --- a/tests/submit_recovery_share.sh +++ b/tests/submit_recovery_share.sh @@ -4,11 +4,6 @@ set -e -function cleanup() { - rm -f member_enc_privk.raw network_enc_pubk.raw -} -trap cleanup 0 - function usage() { echo "Usage:""" @@ -67,21 +62,24 @@ resp=$(curl -sS --fail -X GET https://"${node_rpc_address}"/members/recovery_sha encrypted_share="$(echo "${resp}" | jq -r .encrypted_recovery_share)" nonce="$(echo "${resp}" | jq -r .nonce)" - -## TODO: Mkstemp - +# Temporary directory for raw keys +tmp_dir=$(mktemp -d) +function cleanup() { + rm -rf "${tmp_dir}" +} +trap cleanup EXIT # Parse raw private key from SubjectPublicKeyInfo DER format, as generated by keygenerator.sh --gen-enc-key der_header_privk_len=14 -openssl asn1parse -in "${member_enc_privk}" -strparse "${der_header_privk_len}" -out member_enc_privk.raw -noout +openssl asn1parse -in "${member_enc_privk}" -strparse "${der_header_privk_len}" -out "${tmp_dir}/member_enc_privk.raw" -noout # Parse raw public key generated by network der_header_pubk_len=9 -openssl asn1parse -in "${network_enc_pubk}" -i -strparse "${der_header_pubk_len}" -out network_enc_pubk.raw -noout +openssl asn1parse -in "${network_enc_pubk}" -i -strparse "${der_header_pubk_len}" -out "${tmp_dir}/network_enc_pubk.raw" -noout # step CLI base64 standard to URL encoding for input encrypted_share_b64_url=$(echo "${encrypted_share}" | sed 's/+/-/g; s/\//_/g') # Decrypt encrypted share with nonce, member private key and defunct network public key and submit share # All in one line so that the recovery share is not exposed -echo "${encrypted_share_b64_url}" | step crypto nacl box open base64:"${nonce}" network_enc_pubk.raw member_enc_privk.raw | openssl base64 -A | curl -i -sS --fail https://"${node_rpc_address}"/members/recovery_share/submit "${@}" -d @- \ No newline at end of file +echo "${encrypted_share_b64_url}" | step crypto nacl box open base64:"${nonce}" "${tmp_dir}/network_enc_pubk.raw" "${tmp_dir}/member_enc_privk.raw" | openssl base64 -A | curl -i -sS --fail https://"${node_rpc_address}"/members/recovery_share/submit "${@}" -d @- \ No newline at end of file From 902499833e71f7af659c29f66e8f74833207daab Mon Sep 17 00:00:00 2001 From: Julien Maffre Date: Tue, 16 Jun 2020 13:48:11 +0100 Subject: [PATCH 33/33] Schema --- doc/schemas/recovery_share/submit_params.json | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/doc/schemas/recovery_share/submit_params.json b/doc/schemas/recovery_share/submit_params.json index e312316f10ea..49c61c6fc795 100644 --- a/doc/schemas/recovery_share/submit_params.json +++ b/doc/schemas/recovery_share/submit_params.json @@ -1,13 +1,5 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "properties": { - "recovery_share": { - "type": "string" - } - }, - "required": [ - "recovery_share" - ], "title": "recovery_share/submit/params", - "type": "object" + "type": "string" } \ No newline at end of file