From 2670a8d4cd32d188e3923d3159d951728e5f2460 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 8 Mar 2021 18:16:31 +0000 Subject: [PATCH 01/14] towards js historical endpoints --- src/apps/js_generic/js_generic.cpp | 108 ++++++++++++++++++++++++++--- src/node/rpc/endpoint.h | 7 +- 2 files changed, 104 insertions(+), 11 deletions(-) diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 369c22545570..7ad3244478e8 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -765,6 +765,7 @@ namespace ccfapp { private: NetworkTables& network; + ccfapp::AbstractNodeContext& node_context; JSClassDef kv_class_def = {}; JSClassExoticMethods kv_exotic_methods = {}; @@ -775,7 +776,7 @@ namespace ccfapp metrics::Tracker metrics_tracker; - static JSValue create_ccf_obj(EndpointContext& args, JSContext* ctx) + static JSValue create_ccf_obj(kv::Tx& tx, JSContext* ctx) { auto ccf = JS_NewObject(ctx); @@ -813,7 +814,7 @@ namespace ccfapp JS_NewCFunction(ctx, ccfapp::js_wrap_key, "wrapKey", 3)); auto kv = JS_NewObjectClass(ctx, kv_class_id); - JS_SetOpaque(kv, &args.tx); + JS_SetOpaque(kv, &tx); JS_SetPropertyStr(ctx, ccf, "kv", kv); return ccf; @@ -829,12 +830,12 @@ namespace ccfapp return console; } - static void populate_global_obj(EndpointContext& args, JSContext* ctx) + static void populate_global_obj(kv::Tx& tx, JSContext* ctx) { auto global_obj = JS_GetGlobalObject(ctx); JS_SetPropertyStr(ctx, global_obj, "console", create_console_obj(ctx)); - JS_SetPropertyStr(ctx, global_obj, "ccf", create_ccf_obj(args, ctx)); + JS_SetPropertyStr(ctx, global_obj, "ccf", create_ccf_obj(tx, ctx)); JS_FreeValue(ctx, global_obj); } @@ -994,6 +995,94 @@ namespace ccfapp const std::string& method, const ccf::RESTVerb& verb, EndpointContext& args) + { + // Is this a historical endpoint? + auto endpoints = args.tx.ro(ccf::Tables::ENDPOINTS); + auto info = endpoints->get(ccf::endpoints::EndpointKey{method, verb}); + + if (info.value().historical) + { + // TODO avoid duplication with src/node/historical_queries_adapter.h + // TODO update to latest code with receipts + ccf::TxID target_tx_id; + const auto tx_id_header = args.rpc_ctx->get_request_header(http::headers::CCF_TX_ID); + const auto tx_id_opt = ccf::TxID::from_str(tx_id_header.value()); + target_tx_id = tx_id_opt.value(); + + auto is_tx_committed = [this]( + kv::Consensus::View view, + kv::Consensus::SeqNo seqno, + std::string& error_reason) { + if (consensus == nullptr) + { + error_reason = "Node is not fully configured"; + return false; + } + + const auto tx_view = consensus->get_view(seqno); + const auto committed_seqno = consensus->get_committed_seqno(); + const auto committed_view = consensus->get_view(committed_seqno); + + const auto tx_status = ccf::evaluate_tx_status( + view, seqno, tx_view, committed_view, committed_seqno); + if (tx_status != ccf::TxStatus::Committed) + { + error_reason = fmt::format( + "Only committed transactions can be queried. Transaction {}.{} is " + "{}", + view, + seqno, + ccf::tx_status_to_str(tx_status)); + return false; + } + + return true; + }; + + // Check that the requested transaction ID is available + { + auto error_reason = fmt::format( + "Transaction {} is not available.", target_tx_id.to_str()); + if (!is_tx_committed(target_tx_id.view, target_tx_id.seqno, error_reason)) + { + args.rpc_ctx->set_error( + HTTP_STATUS_BAD_REQUEST, + ccf::errors::TransactionNotFound, + std::move(error_reason)); + return; + } + } + + const auto historic_request_handle = target_tx_id.seqno; + auto& state_cache = node_context.get_historical_state(); + auto historical_store = state_cache.get_store_at(historic_request_handle, target_tx_id.seqno); + if (historical_store == nullptr) + { + args.rpc_ctx->set_response_status(HTTP_STATUS_ACCEPTED); + static constexpr size_t retry_after_seconds = 3; + args.rpc_ctx->set_response_header( + http::headers::RETRY_AFTER, retry_after_seconds); + args.rpc_ctx->set_response_header( + http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); + args.rpc_ctx->set_response_body(fmt::format( + "Historical transaction {} is not currently available.", + target_tx_id.to_str())); + return; + } + auto tx = historical_store->create_tx(); + do_execute_request(method, verb, args, tx); + } + else + { + do_execute_request(method, verb, args, args.tx); + } + } + + void do_execute_request( + const std::string& method, + const ccf::RESTVerb& verb, + EndpointContext& args, + kv::Tx& target_tx) { const auto local_method = method.substr(method.find_first_not_of('/')); @@ -1081,7 +1170,7 @@ namespace ccfapp JS_SetClassProto(ctx, body_class_id, body_proto); // Populate globalThis with console and ccf globals - populate_global_obj(args, ctx); + populate_global_obj(target_tx, ctx); // Compile module if (!handler_script.value().text.has_value()) @@ -1305,9 +1394,10 @@ namespace ccfapp {}; public: - JSHandlers(NetworkTables& network, ccf::AbstractNodeState& node_state) : - UserEndpointRegistry(node_state), - network(network) + JSHandlers(NetworkTables& network, ccfapp::AbstractNodeContext& node_context) : + UserEndpointRegistry(node_context.get_node_state()), + network(network), + node_context(node_context) { JS_NewClassID(&kv_class_id); kv_exotic_methods.get_own_property = js_kv_lookup; @@ -1504,7 +1594,7 @@ namespace ccfapp public: JS(NetworkTables& network, ccfapp::AbstractNodeContext& node_context) : ccf::UserRpcFrontend(*network.tables, js_handlers), - js_handlers(network, node_context.get_node_state()) + js_handlers(network, node_context) {} }; diff --git a/src/node/rpc/endpoint.h b/src/node/rpc/endpoint.h index 0d6e2ead0d21..2c6c87b52ac0 100644 --- a/src/node/rpc/endpoint.h +++ b/src/node/rpc/endpoint.h @@ -73,18 +73,21 @@ namespace ccf nlohmann::json openapi; bool openapi_hidden = false; + bool historical = false; + MSGPACK_DEFINE( forwarding_required, execute_outside_consensus, authn_policies, openapi, - openapi_hidden); + openapi_hidden, + historical); }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EndpointProperties); DECLARE_JSON_REQUIRED_FIELDS( EndpointProperties, forwarding_required, authn_policies); - DECLARE_JSON_OPTIONAL_FIELDS(EndpointProperties, openapi, openapi_hidden); + DECLARE_JSON_OPTIONAL_FIELDS(EndpointProperties, openapi, openapi_hidden, historical); struct EndpointDefinition { From 131d00223e7d9d7fff911707bc7e4255162d9a34 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Tue, 9 Mar 2021 12:04:09 +0000 Subject: [PATCH 02/14] towards historical js endpoints --- samples/apps/forum/src/types/ccf.ts | 26 ++++++++++++++++++++++++++ samples/apps/logging/js/app.json | 12 ++++++++++++ samples/apps/logging/js/src/logging.js | 4 ++++ src/apps/js_generic/js_generic.cpp | 16 ++++++++++++++++ src/kv/tx.h | 15 +++++++++++++++ tests/e2e_logging.py | 11 +++-------- tests/npm-app/src/types/ccf.ts | 21 +++++++++++++++++++++ 7 files changed, 97 insertions(+), 8 deletions(-) diff --git a/samples/apps/forum/src/types/ccf.ts b/samples/apps/forum/src/types/ccf.ts index 01d614ca74e6..ea1f84915a92 100644 --- a/samples/apps/forum/src/types/ccf.ts +++ b/samples/apps/forum/src/types/ccf.ts @@ -48,6 +48,26 @@ export interface KVMap { export type KVMaps = { [key: string]: KVMap }; +export interface ProofElement { + left?: string + right?: string +} + +export type Proof = ProofElement[]; + +export interface Receipt { + signature: string; + root: string; + proof: Proof; + leaf: string; + nodeId: string; +} + +export interface State { + transactionId: string; + receipt?: Receipt; +} + interface WrapAlgoParams { name: string; } @@ -61,6 +81,11 @@ export interface AESKWPParams extends WrapAlgoParams { name: "AES-KWP"; } +export interface RsaOaepAESKWPParams extends WrapAlgoParams { + name: "RSA-OAEP-AES-KWP"; + label?: ArrayBuffer; +} + export interface CCF { strToBuf(v: string): ArrayBuffer; bufToStr(v: ArrayBuffer): string; @@ -74,6 +99,7 @@ export interface CCF { ): ArrayBuffer; kv: KVMaps; + state: State; } export const ccf = globalThis.ccf as CCF; diff --git a/samples/apps/logging/js/app.json b/samples/apps/logging/js/app.json index 2318ae0077dd..afd7f0f4a6c3 100644 --- a/samples/apps/logging/js/app.json +++ b/samples/apps/logging/js/app.json @@ -29,6 +29,18 @@ "openapi": {} } }, + "/log/private/historical": { + "get": { + "js_module": "logging.js", + "js_function": "get_historical", + "forwarding_required": "never", + "execute_outside_consensus": "never", + "authn_policies": ["jwt", "user_cert"], + "historical": true, + "readonly": true, + "openapi": {} + } + }, "/log/public": { "get": { "js_module": "logging.js", diff --git a/samples/apps/logging/js/src/logging.js b/samples/apps/logging/js/src/logging.js index 1ec039c13377..401b7956c399 100644 --- a/samples/apps/logging/js/src/logging.js +++ b/samples/apps/logging/js/src/logging.js @@ -29,6 +29,10 @@ export function get_private(request) { return get_record(ccf.kv["records"], id); } +export function get_historical(request) { + return get_private(request); +} + export function get_public(request) { const id = get_id_from_request_query(request); return get_record(ccf.kv["public:records"], id); diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 7ad3244478e8..17f1c2fff7a7 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -817,6 +817,19 @@ namespace ccfapp JS_SetOpaque(kv, &tx); JS_SetPropertyStr(ctx, ccf, "kv", kv); + auto state = JS_NewObject(ctx); + auto kv_tx_id = tx.get_read_tx_id(); + ccf::TxID tx_id; + tx_id.view = static_cast(kv_tx_id.term); + tx_id.seqno = static_cast(kv_tx_id.version); + JS_SetPropertyStr( + ctx, + state, + "transactionId", + JS_NewString(ctx, tx_id.to_str().c_str())); + // TODO receipt + JS_SetPropertyStr(ctx, ccf, "state", state); + return ccf; } @@ -1000,6 +1013,9 @@ namespace ccfapp auto endpoints = args.tx.ro(ccf::Tables::ENDPOINTS); auto info = endpoints->get(ccf::endpoints::EndpointKey{method, verb}); + if (!info.has_value()) { + throw std::logic_error("no endpoint info found"); + } if (info.value().historical) { // TODO avoid duplication with src/node/historical_queries_adapter.h diff --git a/src/kv/tx.h b/src/kv/tx.h index 75a4e1597b10..7663dc3d8a02 100644 --- a/src/kv/tx.h +++ b/src/kv/tx.h @@ -242,6 +242,21 @@ namespace kv return term; } + TxID get_read_tx_id() + { + if (read_version != NoVersion) + { + TxID tx_id; + tx_id.term = term; + tx_id.version = read_version; + return tx_id; + } + else + { + throw std::logic_error("Read version not set"); + } + } + void set_read_version_and_term(Version v, Term t) { if (read_version == NoVersion) diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index 798a326e98dc..27726939aba4 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -422,14 +422,9 @@ def test_historical_query(network, args): LOG.warning("Skipping historical queries in BFT") return network - if args.package == "liblogging": - network.txs.issue(network, number_txs=2) - network.txs.issue(network, number_txs=2, repeat=True) - network.txs.verify() - else: - LOG.warning( - f"Skipping {inspect.currentframe().f_code.co_name} as application is not C++" - ) + network.txs.issue(network, number_txs=2) + network.txs.issue(network, number_txs=2, repeat=True) + network.txs.verify() return network diff --git a/tests/npm-app/src/types/ccf.ts b/tests/npm-app/src/types/ccf.ts index 31378a815bb7..ea1f84915a92 100644 --- a/tests/npm-app/src/types/ccf.ts +++ b/tests/npm-app/src/types/ccf.ts @@ -48,6 +48,26 @@ export interface KVMap { export type KVMaps = { [key: string]: KVMap }; +export interface ProofElement { + left?: string + right?: string +} + +export type Proof = ProofElement[]; + +export interface Receipt { + signature: string; + root: string; + proof: Proof; + leaf: string; + nodeId: string; +} + +export interface State { + transactionId: string; + receipt?: Receipt; +} + interface WrapAlgoParams { name: string; } @@ -79,6 +99,7 @@ export interface CCF { ): ArrayBuffer; kv: KVMaps; + state: State; } export const ccf = globalThis.ccf as CCF; From 36691fba1464a995d1f5d026c89593b5240019d1 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Tue, 9 Mar 2021 17:05:15 +0000 Subject: [PATCH 03/14] reuse historical query adapter --- src/apps/js_generic/js_generic.cpp | 64 +++++++++--------------------- 1 file changed, 19 insertions(+), 45 deletions(-) diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 1cf1e7b056d3..3663d8a48d5a 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -816,20 +816,24 @@ namespace ccfapp auto kv = JS_NewObjectClass(ctx, kv_class_id); JS_SetOpaque(kv, &tx); JS_SetPropertyStr(ctx, ccf, "kv", kv); - - auto state = JS_NewObject(ctx); +/* + // auto state = JS_NewObject(ctx); auto kv_tx_id = tx.get_read_tx_id(); ccf::TxID tx_id; tx_id.view = static_cast(kv_tx_id.term); tx_id.seqno = static_cast(kv_tx_id.version); + std::string tx_id_str = tx_id.to_str(); + const char* tx_id_cstr = tx_id_str.c_str(); + auto s = JS_NewString(ctx, tx_id_cstr); + // FIXME why does this lead to uncollected garbage?? JS_SetPropertyStr( ctx, - state, + ccf, //state, "transactionId", - JS_NewString(ctx, tx_id.to_str().c_str())); + s); // TODO receipt - JS_SetPropertyStr(ctx, ccf, "state", state); - + // JS_SetPropertyStr(ctx, ccf, "state", state); +*/ return ccf; } @@ -1018,13 +1022,8 @@ namespace ccfapp } if (info.value().historical) { - // TODO avoid duplication with src/node/historical_queries_adapter.h - // TODO update to latest code with receipts - ccf::TxID target_tx_id; - const auto tx_id_header = args.rpc_ctx->get_request_header(http::headers::CCF_TX_ID); - const auto tx_id_opt = ccf::TxID::from_str(tx_id_header.value()); - target_tx_id = tx_id_opt.value(); - + // TODO doesn't really belong here + // (copied from samples/apps/logging/logging.cpp) auto is_tx_committed = [this]( kv::Consensus::View view, kv::Consensus::SeqNo seqno, @@ -1055,38 +1054,13 @@ namespace ccfapp return true; }; - // Check that the requested transaction ID is available - { - auto error_reason = fmt::format( - "Transaction {} is not available.", target_tx_id.to_str()); - if (!is_tx_committed(target_tx_id.view, target_tx_id.seqno, error_reason)) - { - args.rpc_ctx->set_error( - HTTP_STATUS_BAD_REQUEST, - ccf::errors::TransactionNotFound, - std::move(error_reason)); - return; - } - } - - const auto historic_request_handle = target_tx_id.seqno; - auto& state_cache = context.get_historical_state(); - auto historical_store = state_cache.get_store_at(historic_request_handle, target_tx_id.seqno); - if (historical_store == nullptr) - { - args.rpc_ctx->set_response_status(HTTP_STATUS_ACCEPTED); - static constexpr size_t retry_after_seconds = 3; - args.rpc_ctx->set_response_header( - http::headers::RETRY_AFTER, retry_after_seconds); - args.rpc_ctx->set_response_header( - http::headers::CONTENT_TYPE, http::headervalues::contenttype::TEXT); - args.rpc_ctx->set_response_body(fmt::format( - "Historical transaction {} is not currently available.", - target_tx_id.to_str())); - return; - } - auto tx = historical_store->create_tx(); - do_execute_request(method, verb, args, tx); + ccf::historical::adapter([this, &method, &verb](ccf::EndpointContext& args, ccf::historical::StatePtr state) { + auto tx = state->store->create_tx(); + // TODO forward receipt + auto receipt = state->receipt; + do_execute_request(method, verb, args, tx); + }, + context.get_historical_state(), is_tx_committed)(args); } else { From 236791600fb07e2cbd12505f5d3d73bfe5bed694 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Tue, 9 Mar 2021 17:55:41 +0000 Subject: [PATCH 04/14] expose receipt to js --- samples/apps/logging/js/app.json | 12 ++++ samples/apps/logging/js/src/logging.js | 6 ++ src/apps/js_generic/js_generic.cpp | 95 +++++++++++++++++++++----- 3 files changed, 95 insertions(+), 18 deletions(-) diff --git a/samples/apps/logging/js/app.json b/samples/apps/logging/js/app.json index afd7f0f4a6c3..53e218d5469c 100644 --- a/samples/apps/logging/js/app.json +++ b/samples/apps/logging/js/app.json @@ -41,6 +41,18 @@ "openapi": {} } }, + "/log/private/historical_receipt": { + "get": { + "js_module": "logging.js", + "js_function": "get_historical_with_receipt", + "forwarding_required": "never", + "execute_outside_consensus": "never", + "authn_policies": ["jwt", "user_cert"], + "historical": true, + "readonly": true, + "openapi": {} + } + }, "/log/public": { "get": { "js_module": "logging.js", diff --git a/samples/apps/logging/js/src/logging.js b/samples/apps/logging/js/src/logging.js index 401b7956c399..509bba38891e 100644 --- a/samples/apps/logging/js/src/logging.js +++ b/samples/apps/logging/js/src/logging.js @@ -33,6 +33,12 @@ export function get_historical(request) { return get_private(request); } +export function get_historical_with_receipt(request) { + const result = get_private(request); + result.body.receipt = ccf.state.receipt; + return result; +} + export function get_public(request) { const id = get_id_from_request_query(request); return get_record(ccf.kv["public:records"], id); diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 3663d8a48d5a..96c4210e650a 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -776,7 +776,7 @@ namespace ccfapp metrics::Tracker metrics_tracker; - static JSValue create_ccf_obj(kv::Tx& tx, JSContext* ctx) + static JSValue create_ccf_obj(kv::Tx& tx, historical::TxReceiptPtr receipt, JSContext* ctx) { auto ccf = JS_NewObject(ctx); @@ -816,24 +816,83 @@ namespace ccfapp auto kv = JS_NewObjectClass(ctx, kv_class_id); JS_SetOpaque(kv, &tx); JS_SetPropertyStr(ctx, ccf, "kv", kv); -/* - // auto state = JS_NewObject(ctx); + + auto state = JS_NewObject(ctx); auto kv_tx_id = tx.get_read_tx_id(); ccf::TxID tx_id; tx_id.view = static_cast(kv_tx_id.term); tx_id.seqno = static_cast(kv_tx_id.version); - std::string tx_id_str = tx_id.to_str(); - const char* tx_id_cstr = tx_id_str.c_str(); - auto s = JS_NewString(ctx, tx_id_cstr); // FIXME why does this lead to uncollected garbage?? JS_SetPropertyStr( ctx, - ccf, //state, + state, "transactionId", - s); - // TODO receipt - // JS_SetPropertyStr(ctx, ccf, "state", state); -*/ + JS_NewString(ctx, tx_id.to_str().c_str())); + + if (receipt) + { + ccf::GetReceipt::Out receipt_out; + receipt_out.from_receipt(receipt); + auto js_receipt = JS_NewObject(ctx); + JS_SetPropertyStr( + ctx, + js_receipt, + "signature", + JS_NewString(ctx, receipt_out.signature.c_str()) + ); + JS_SetPropertyStr( + ctx, + js_receipt, + "root", + JS_NewString(ctx, receipt_out.root.c_str()) + ); + JS_SetPropertyStr( + ctx, + js_receipt, + "leaf", + JS_NewString(ctx, receipt_out.leaf.c_str()) + ); + JS_SetPropertyStr( + ctx, + js_receipt, + "node_id", + JS_NewString(ctx, receipt_out.node_id.value().c_str()) + ); + auto proof = JS_NewArray(ctx); + uint32_t i = 0; + for (auto& element : receipt_out.proof) + { + auto js_element = JS_NewObject(ctx); + auto is_left = element.left.has_value(); + JS_SetPropertyStr( + ctx, + js_element, + is_left ? "left" : "right", + JS_NewString(ctx, (is_left ? element.left : element.right).value().c_str()) + ); + JS_DefinePropertyValueUint32( + ctx, + proof, + i++, + js_element, + JS_PROP_C_W_E); + } + JS_SetPropertyStr( + ctx, + js_receipt, + "proof", + proof + ); + + JS_SetPropertyStr( + ctx, + state, + "receipt", + js_receipt); + } + + JS_SetPropertyStr(ctx, ccf, "state", state); + return ccf; } @@ -847,12 +906,12 @@ namespace ccfapp return console; } - static void populate_global_obj(kv::Tx& tx, JSContext* ctx) + static void populate_global_obj(kv::Tx& tx, ccf::historical::TxReceiptPtr receipt, JSContext* ctx) { auto global_obj = JS_GetGlobalObject(ctx); JS_SetPropertyStr(ctx, global_obj, "console", create_console_obj(ctx)); - JS_SetPropertyStr(ctx, global_obj, "ccf", create_ccf_obj(tx, ctx)); + JS_SetPropertyStr(ctx, global_obj, "ccf", create_ccf_obj(tx, receipt, ctx)); JS_FreeValue(ctx, global_obj); } @@ -1056,15 +1115,14 @@ namespace ccfapp ccf::historical::adapter([this, &method, &verb](ccf::EndpointContext& args, ccf::historical::StatePtr state) { auto tx = state->store->create_tx(); - // TODO forward receipt auto receipt = state->receipt; - do_execute_request(method, verb, args, tx); + do_execute_request(method, verb, args, tx, receipt); }, context.get_historical_state(), is_tx_committed)(args); } else { - do_execute_request(method, verb, args, args.tx); + do_execute_request(method, verb, args, args.tx, nullptr); } } @@ -1072,7 +1130,8 @@ namespace ccfapp const std::string& method, const ccf::RESTVerb& verb, EndpointContext& args, - kv::Tx& target_tx) + kv::Tx& target_tx, + ccf::historical::TxReceiptPtr receipt) { const auto local_method = method.substr(method.find_first_not_of('/')); @@ -1160,7 +1219,7 @@ namespace ccfapp JS_SetClassProto(ctx, body_class_id, body_proto); // Populate globalThis with console and ccf globals - populate_global_obj(target_tx, ctx); + populate_global_obj(target_tx, receipt, ctx); // Compile module if (!handler_script.value().text.has_value()) From ef2a32a7a2a16ed3239cad9e67a53326a23ab720 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Tue, 9 Mar 2021 17:56:00 +0000 Subject: [PATCH 05/14] temporarily disable all except historical tests --- tests/e2e_logging.py | 114 ++++++++++++++++++++----------------------- 1 file changed, 54 insertions(+), 60 deletions(-) diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index bb56fed2f09e..d34659ce62f2 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -441,40 +441,34 @@ def test_historical_receipts(network, args): LOG.warning("Skipping historical queries in BFT") return network - if args.package == "liblogging": - primary, backups = network.find_nodes() - cert_path = os.path.join(primary.common_dir, f"{primary.local_node_id}.pem") - with open(cert_path) as c: - primary_cert = load_pem_x509_certificate( - c.read().encode("ascii"), default_backend() - ) + primary, backups = network.find_nodes() + cert_path = os.path.join(primary.common_dir, f"{primary.local_node_id}.pem") + with open(cert_path) as c: + primary_cert = load_pem_x509_certificate( + c.read().encode("ascii"), default_backend() + ) - TXS_COUNT = 5 - network.txs.issue(network, number_txs=5) - for idx in range(1, TXS_COUNT + 1): - for node in [primary, backups[0]]: - first_msg = network.txs.priv[idx][0] - first_receipt = network.txs.get_receipt( - node, idx, first_msg["seqno"], first_msg["view"] - ) - r = first_receipt.json()["receipt"] - assert r["root"] == ccf.receipt.root(r["leaf"], r["proof"]) - ccf.receipt.verify(r["root"], r["signature"], primary_cert) - - # receipt.verify() raises if it fails, but does not return anything - verified = True - try: - ccf.receipt.verify( - hashlib.sha256(b"").hexdigest(), r["signature"], primary_cert + TXS_COUNT = 5 + network.txs.issue(network, number_txs=5) + for idx in range(1, TXS_COUNT + 1): + for node in [primary, backups[0]]: + first_msg = network.txs.priv[idx][0] + first_receipt = network.txs.get_receipt( + node, idx, first_msg["seqno"], first_msg["view"] ) - except InvalidSignature: - verified = False - assert not verified - - else: - LOG.warning( - f"Skipping {inspect.currentframe().f_code.co_name} as application is not C++" + r = first_receipt.json()["receipt"] + assert r["root"] == ccf.receipt.root(r["leaf"], r["proof"]) + ccf.receipt.verify(r["root"], r["signature"], primary_cert) + + # receipt.verify() raises if it fails, but does not return anything + verified = True + try: + ccf.receipt.verify( + hashlib.sha256(b"").hexdigest(), r["signature"], primary_cert ) + except InvalidSignature: + verified = False + assert not verified return network @@ -1036,36 +1030,36 @@ def run(args): ) as network: network.start_and_join(args) - network = test( - network, - args, - verify=args.package != "libjs_generic", - ) - network = test_illegal(network, args, verify=args.package != "libjs_generic") - network = test_large_messages(network, args) - network = test_remove(network, args) - network = test_forwarding_frontends(network, args) - network = test_user_data_ACL(network, args) - network = test_cert_prefix(network, args) - network = test_anonymous_caller(network, args) - network = test_multi_auth(network, args) - network = test_custom_auth(network, args) - network = test_raw_text(network, args) + # network = test( + # network, + # args, + # verify=args.package != "libjs_generic", + # ) + # network = test_illegal(network, args, verify=args.package != "libjs_generic") + # network = test_large_messages(network, args) + # network = test_remove(network, args) + # network = test_forwarding_frontends(network, args) + # network = test_user_data_ACL(network, args) + # network = test_cert_prefix(network, args) + # network = test_anonymous_caller(network, args) + # network = test_multi_auth(network, args) + # network = test_custom_auth(network, args) + # network = test_raw_text(network, args) network = test_historical_query(network, args) - network = test_historical_query_range(network, args) - network = test_view_history(network, args) - network = test_primary(network, args) - network = test_network_node_info(network, args) - network = test_metrics(network, args) - network = test_memory(network, args) - # BFT does not handle re-keying yet - if args.consensus == "cft": - network = test_liveness(network, args) - network = test_rekey(network, args) - network = test_liveness(network, args) - if args.package == "liblogging": - network = test_ws(network, args) - network = test_receipts(network, args) + # network = test_historical_query_range(network, args) + # network = test_view_history(network, args) + # network = test_primary(network, args) + # network = test_network_node_info(network, args) + # network = test_metrics(network, args) + # network = test_memory(network, args) + # # BFT does not handle re-keying yet + # if args.consensus == "cft": + # network = test_liveness(network, args) + # network = test_rekey(network, args) + # network = test_liveness(network, args) + # if args.package == "liblogging": + # network = test_ws(network, args) + # network = test_receipts(network, args) network = test_historical_receipts(network, args) From 7e81b601186527cb44b306513ed61d7fe3d48cb1 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Tue, 9 Mar 2021 18:01:02 +0000 Subject: [PATCH 06/14] node_id -> nodeId --- src/apps/js_generic/js_generic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 96c4210e650a..57e3679fe55f 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -855,7 +855,7 @@ namespace ccfapp JS_SetPropertyStr( ctx, js_receipt, - "node_id", + "nodeId", JS_NewString(ctx, receipt_out.node_id.value().c_str()) ); auto proof = JS_NewArray(ctx); From 182b63def344cddc5b851553da2a83ebf84a827c Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Wed, 10 Mar 2021 17:12:26 +0000 Subject: [PATCH 07/14] fixes --- samples/apps/forum/src/types/ccf.ts | 4 +- samples/apps/logging/js/src/logging.js | 8 +- src/apps/js_generic/js_generic.cpp | 132 ++++++++++++------------- src/kv/tx.h | 15 --- src/node/rpc/endpoint.h | 3 +- tests/e2e_logging.py | 58 +++++------ tests/npm-app/src/types/ccf.ts | 4 +- 7 files changed, 102 insertions(+), 122 deletions(-) diff --git a/samples/apps/forum/src/types/ccf.ts b/samples/apps/forum/src/types/ccf.ts index ea1f84915a92..80dc3a8f6ed9 100644 --- a/samples/apps/forum/src/types/ccf.ts +++ b/samples/apps/forum/src/types/ccf.ts @@ -49,8 +49,8 @@ export interface KVMap { export type KVMaps = { [key: string]: KVMap }; export interface ProofElement { - left?: string - right?: string + left?: string; + right?: string; } export type Proof = ProofElement[]; diff --git a/samples/apps/logging/js/src/logging.js b/samples/apps/logging/js/src/logging.js index 509bba38891e..a13077e7b163 100644 --- a/samples/apps/logging/js/src/logging.js +++ b/samples/apps/logging/js/src/logging.js @@ -30,13 +30,13 @@ export function get_private(request) { } export function get_historical(request) { - return get_private(request); + return get_private(request); } export function get_historical_with_receipt(request) { - const result = get_private(request); - result.body.receipt = ccf.state.receipt; - return result; + const result = get_private(request); + result.body.receipt = ccf.state.receipt; + return result; } export function get_public(request) { diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 57e3679fe55f..dba4770efcf4 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -568,7 +568,9 @@ namespace ccfapp JSValueConst this_val, JSAtom property) { - const auto property_name = JS_AtomToCString(ctx, property); + const auto property_name_c = JS_AtomToCString(ctx, property); + const auto property_name = std::string(property_name_c); + JS_FreeCString(ctx, property_name_c); LOG_TRACE_FMT("Looking for kv map '{}'", property_name); const auto [security_domain, access_category] = @@ -776,7 +778,11 @@ namespace ccfapp metrics::Tracker metrics_tracker; - static JSValue create_ccf_obj(kv::Tx& tx, historical::TxReceiptPtr receipt, JSContext* ctx) + static JSValue create_ccf_obj( + kv::Tx& tx, + const std::optional& transaction_id, + historical::TxReceiptPtr receipt, + JSContext* ctx) { auto ccf = JS_NewObject(ctx); @@ -817,20 +823,20 @@ namespace ccfapp JS_SetOpaque(kv, &tx); JS_SetPropertyStr(ctx, ccf, "kv", kv); - auto state = JS_NewObject(ctx); - auto kv_tx_id = tx.get_read_tx_id(); - ccf::TxID tx_id; - tx_id.view = static_cast(kv_tx_id.term); - tx_id.seqno = static_cast(kv_tx_id.version); - // FIXME why does this lead to uncollected garbage?? - JS_SetPropertyStr( - ctx, - state, - "transactionId", - JS_NewString(ctx, tx_id.to_str().c_str())); - + // Historical queries if (receipt) { + auto state = JS_NewObject(ctx); + + ccf::TxID tx_id; + tx_id.seqno = static_cast(transaction_id.value().version); + tx_id.view = static_cast(transaction_id.value().term); + JS_SetPropertyStr( + ctx, + state, + "transactionId", + JS_NewString(ctx, tx_id.to_str().c_str())); + ccf::GetReceipt::Out receipt_out; receipt_out.from_receipt(receipt); auto js_receipt = JS_NewObject(ctx); @@ -838,26 +844,16 @@ namespace ccfapp ctx, js_receipt, "signature", - JS_NewString(ctx, receipt_out.signature.c_str()) - ); + JS_NewString(ctx, receipt_out.signature.c_str())); JS_SetPropertyStr( - ctx, - js_receipt, - "root", - JS_NewString(ctx, receipt_out.root.c_str()) - ); + ctx, js_receipt, "root", JS_NewString(ctx, receipt_out.root.c_str())); JS_SetPropertyStr( - ctx, - js_receipt, - "leaf", - JS_NewString(ctx, receipt_out.leaf.c_str()) - ); + ctx, js_receipt, "leaf", JS_NewString(ctx, receipt_out.leaf.c_str())); JS_SetPropertyStr( ctx, js_receipt, "nodeId", - JS_NewString(ctx, receipt_out.node_id.value().c_str()) - ); + JS_NewString(ctx, receipt_out.node_id.value().c_str())); auto proof = JS_NewArray(ctx); uint32_t i = 0; for (auto& element : receipt_out.proof) @@ -868,30 +864,17 @@ namespace ccfapp ctx, js_element, is_left ? "left" : "right", - JS_NewString(ctx, (is_left ? element.left : element.right).value().c_str()) - ); + JS_NewString( + ctx, (is_left ? element.left : element.right).value().c_str())); JS_DefinePropertyValueUint32( - ctx, - proof, - i++, - js_element, - JS_PROP_C_W_E); + ctx, proof, i++, js_element, JS_PROP_C_W_E); } - JS_SetPropertyStr( - ctx, - js_receipt, - "proof", - proof - ); - - JS_SetPropertyStr( - ctx, - state, - "receipt", - js_receipt); - } + JS_SetPropertyStr(ctx, js_receipt, "proof", proof); - JS_SetPropertyStr(ctx, ccf, "state", state); + JS_SetPropertyStr(ctx, state, "receipt", js_receipt); + + JS_SetPropertyStr(ctx, ccf, "state", state); + } return ccf; } @@ -906,12 +889,20 @@ namespace ccfapp return console; } - static void populate_global_obj(kv::Tx& tx, ccf::historical::TxReceiptPtr receipt, JSContext* ctx) + static void populate_global_obj( + kv::Tx& tx, + const std::optional& transaction_id, + ccf::historical::TxReceiptPtr receipt, + JSContext* ctx) { auto global_obj = JS_GetGlobalObject(ctx); JS_SetPropertyStr(ctx, global_obj, "console", create_console_obj(ctx)); - JS_SetPropertyStr(ctx, global_obj, "ccf", create_ccf_obj(tx, receipt, ctx)); + JS_SetPropertyStr( + ctx, + global_obj, + "ccf", + create_ccf_obj(tx, transaction_id, receipt, ctx)); JS_FreeValue(ctx, global_obj); } @@ -1073,20 +1064,17 @@ namespace ccfapp EndpointContext& args) { // Is this a historical endpoint? - auto endpoints = args.tx.ro(ccf::Tables::ENDPOINTS); + auto endpoints = + args.tx.ro(ccf::Tables::ENDPOINTS); auto info = endpoints->get(ccf::endpoints::EndpointKey{method, verb}); - if (!info.has_value()) { - throw std::logic_error("no endpoint info found"); - } - if (info.value().historical) + if (info.has_value() && info.value().historical) { - // TODO doesn't really belong here // (copied from samples/apps/logging/logging.cpp) auto is_tx_committed = [this]( - kv::Consensus::View view, - kv::Consensus::SeqNo seqno, - std::string& error_reason) { + kv::Consensus::View view, + kv::Consensus::SeqNo seqno, + std::string& error_reason) { if (consensus == nullptr) { error_reason = "Node is not fully configured"; @@ -1102,7 +1090,8 @@ namespace ccfapp if (tx_status != ccf::TxStatus::Committed) { error_reason = fmt::format( - "Only committed transactions can be queried. Transaction {}.{} is " + "Only committed transactions can be queried. Transaction {}.{} " + "is " "{}", view, seqno, @@ -1113,16 +1102,20 @@ namespace ccfapp return true; }; - ccf::historical::adapter([this, &method, &verb](ccf::EndpointContext& args, ccf::historical::StatePtr state) { - auto tx = state->store->create_tx(); - auto receipt = state->receipt; - do_execute_request(method, verb, args, tx, receipt); - }, - context.get_historical_state(), is_tx_committed)(args); + ccf::historical::adapter( + [this, &method, &verb]( + ccf::EndpointContext& args, ccf::historical::StatePtr state) { + auto tx = state->store->create_tx(); + auto tx_id = state->transaction_id; + auto receipt = state->receipt; + do_execute_request(method, verb, args, tx, tx_id, receipt); + }, + context.get_historical_state(), + is_tx_committed)(args); } else { - do_execute_request(method, verb, args, args.tx, nullptr); + do_execute_request(method, verb, args, args.tx, std::nullopt, nullptr); } } @@ -1131,6 +1124,7 @@ namespace ccfapp const ccf::RESTVerb& verb, EndpointContext& args, kv::Tx& target_tx, + const std::optional& transaction_id, ccf::historical::TxReceiptPtr receipt) { const auto local_method = method.substr(method.find_first_not_of('/')); @@ -1219,7 +1213,7 @@ namespace ccfapp JS_SetClassProto(ctx, body_class_id, body_proto); // Populate globalThis with console and ccf globals - populate_global_obj(target_tx, receipt, ctx); + populate_global_obj(target_tx, transaction_id, receipt, ctx); // Compile module if (!handler_script.value().text.has_value()) diff --git a/src/kv/tx.h b/src/kv/tx.h index 7663dc3d8a02..75a4e1597b10 100644 --- a/src/kv/tx.h +++ b/src/kv/tx.h @@ -242,21 +242,6 @@ namespace kv return term; } - TxID get_read_tx_id() - { - if (read_version != NoVersion) - { - TxID tx_id; - tx_id.term = term; - tx_id.version = read_version; - return tx_id; - } - else - { - throw std::logic_error("Read version not set"); - } - } - void set_read_version_and_term(Version v, Term t) { if (read_version == NoVersion) diff --git a/src/node/rpc/endpoint.h b/src/node/rpc/endpoint.h index 2c6c87b52ac0..68b8dd8c99c1 100644 --- a/src/node/rpc/endpoint.h +++ b/src/node/rpc/endpoint.h @@ -87,7 +87,8 @@ namespace ccf DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EndpointProperties); DECLARE_JSON_REQUIRED_FIELDS( EndpointProperties, forwarding_required, authn_policies); - DECLARE_JSON_OPTIONAL_FIELDS(EndpointProperties, openapi, openapi_hidden, historical); + DECLARE_JSON_OPTIONAL_FIELDS( + EndpointProperties, openapi, openapi_hidden, historical); struct EndpointDefinition { diff --git a/tests/e2e_logging.py b/tests/e2e_logging.py index d34659ce62f2..511f34fc75a6 100644 --- a/tests/e2e_logging.py +++ b/tests/e2e_logging.py @@ -1030,36 +1030,36 @@ def run(args): ) as network: network.start_and_join(args) - # network = test( - # network, - # args, - # verify=args.package != "libjs_generic", - # ) - # network = test_illegal(network, args, verify=args.package != "libjs_generic") - # network = test_large_messages(network, args) - # network = test_remove(network, args) - # network = test_forwarding_frontends(network, args) - # network = test_user_data_ACL(network, args) - # network = test_cert_prefix(network, args) - # network = test_anonymous_caller(network, args) - # network = test_multi_auth(network, args) - # network = test_custom_auth(network, args) - # network = test_raw_text(network, args) + network = test( + network, + args, + verify=args.package != "libjs_generic", + ) + network = test_illegal(network, args, verify=args.package != "libjs_generic") + network = test_large_messages(network, args) + network = test_remove(network, args) + network = test_forwarding_frontends(network, args) + network = test_user_data_ACL(network, args) + network = test_cert_prefix(network, args) + network = test_anonymous_caller(network, args) + network = test_multi_auth(network, args) + network = test_custom_auth(network, args) + network = test_raw_text(network, args) network = test_historical_query(network, args) - # network = test_historical_query_range(network, args) - # network = test_view_history(network, args) - # network = test_primary(network, args) - # network = test_network_node_info(network, args) - # network = test_metrics(network, args) - # network = test_memory(network, args) - # # BFT does not handle re-keying yet - # if args.consensus == "cft": - # network = test_liveness(network, args) - # network = test_rekey(network, args) - # network = test_liveness(network, args) - # if args.package == "liblogging": - # network = test_ws(network, args) - # network = test_receipts(network, args) + network = test_historical_query_range(network, args) + network = test_view_history(network, args) + network = test_primary(network, args) + network = test_network_node_info(network, args) + network = test_metrics(network, args) + network = test_memory(network, args) + # BFT does not handle re-keying yet + if args.consensus == "cft": + network = test_liveness(network, args) + network = test_rekey(network, args) + network = test_liveness(network, args) + if args.package == "liblogging": + network = test_ws(network, args) + network = test_receipts(network, args) network = test_historical_receipts(network, args) diff --git a/tests/npm-app/src/types/ccf.ts b/tests/npm-app/src/types/ccf.ts index ea1f84915a92..80dc3a8f6ed9 100644 --- a/tests/npm-app/src/types/ccf.ts +++ b/tests/npm-app/src/types/ccf.ts @@ -49,8 +49,8 @@ export interface KVMap { export type KVMaps = { [key: string]: KVMap }; export interface ProofElement { - left?: string - right?: string + left?: string; + right?: string; } export type Proof = ProofElement[]; From d6a60beae25a0d2d4e9ddd51bd704a15532eb019 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Wed, 10 Mar 2021 17:18:27 +0000 Subject: [PATCH 08/14] . --- samples/apps/forum/src/types/ccf.ts | 4 ++-- tests/npm-app/src/types/ccf.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/apps/forum/src/types/ccf.ts b/samples/apps/forum/src/types/ccf.ts index 80dc3a8f6ed9..6cbb493ccad8 100644 --- a/samples/apps/forum/src/types/ccf.ts +++ b/samples/apps/forum/src/types/ccf.ts @@ -65,7 +65,7 @@ export interface Receipt { export interface State { transactionId: string; - receipt?: Receipt; + receipt: Receipt; } interface WrapAlgoParams { @@ -99,7 +99,7 @@ export interface CCF { ): ArrayBuffer; kv: KVMaps; - state: State; + state?: State; } export const ccf = globalThis.ccf as CCF; diff --git a/tests/npm-app/src/types/ccf.ts b/tests/npm-app/src/types/ccf.ts index 80dc3a8f6ed9..6cbb493ccad8 100644 --- a/tests/npm-app/src/types/ccf.ts +++ b/tests/npm-app/src/types/ccf.ts @@ -65,7 +65,7 @@ export interface Receipt { export interface State { transactionId: string; - receipt?: Receipt; + receipt: Receipt; } interface WrapAlgoParams { @@ -99,7 +99,7 @@ export interface CCF { ): ArrayBuffer; kv: KVMaps; - state: State; + state?: State; } export const ccf = globalThis.ccf as CCF; From a77097517aced16593e4af02771d1520eb768b1b Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Thu, 11 Mar 2021 09:24:46 +0000 Subject: [PATCH 09/14] Update src/apps/js_generic/js_generic.cpp Co-authored-by: Amaury Chamayou --- src/apps/js_generic/js_generic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 00149a99eaed..d1ec9d233ae2 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -569,7 +569,7 @@ namespace ccfapp JSAtom property) { const auto property_name_c = JS_AtomToCString(ctx, property); - const auto property_name = std::string(property_name_c); + const std::string property_name(property_name_c); JS_FreeCString(ctx, property_name_c); LOG_TRACE_FMT("Looking for kv map '{}'", property_name); From b24dfc3293d51783da10c54ce0f1df98ab7dcf04 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Thu, 11 Mar 2021 10:49:11 +0000 Subject: [PATCH 10/14] pr feedback --- CHANGELOG.md | 6 +++++ cmake/quickjs.cmake | 10 +++++--- samples/apps/forum/src/types/ccf.ts | 4 ++-- samples/apps/logging/js/src/logging.js | 2 +- samples/apps/logging/logging.cpp | 26 ++------------------ src/apps/js_generic/js_generic.cpp | 28 ++-------------------- src/node/historical_queries_adapter.h | 33 ++++++++++++++++++++++++++ tests/npm-app/src/types/ccf.ts | 4 ++-- 8 files changed, 55 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dcab163c99ce..26250aa330f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Single-transaction historical query support has been added to JavaScript endpoints (#2285). + ## [0.19.0] ### Changed diff --git a/cmake/quickjs.cmake b/cmake/quickjs.cmake index 0098d8a10b9b..a43b5bb91fc0 100644 --- a/cmake/quickjs.cmake +++ b/cmake/quickjs.cmake @@ -28,8 +28,10 @@ if("sgx" IN_LIST COMPILE_TARGETS) quickjs.enclave STATIC ${QUICKJS_SRC} ${CCF_DIR}/3rdparty/stub/time.c ) target_compile_options( - quickjs.enclave PUBLIC -nostdinc -DCONFIG_VERSION="${QUICKJS_VERSION}" - -DEMSCRIPTEN -DCONFIG_STACK_CHECK + quickjs.enclave + PUBLIC -nostdinc -DCONFIG_VERSION="${QUICKJS_VERSION}" -DEMSCRIPTEN + -DCONFIG_STACK_CHECK + PRIVATE $<$:-DDUMP_LEAKS> ) target_link_libraries(quickjs.enclave PUBLIC ${OE_TARGET_LIBC}) set_property(TARGET quickjs.enclave PROPERTY POSITION_INDEPENDENT_CODE ON) @@ -40,7 +42,9 @@ endif() add_library(quickjs.host STATIC ${QUICKJS_SRC}) target_compile_options( - quickjs.host PUBLIC -DCONFIG_VERSION="${QUICKJS_VERSION}" + quickjs.host + PUBLIC -DCONFIG_VERSION="${QUICKJS_VERSION}" + PRIVATE $<$:-DDUMP_LEAKS> ) add_san(quickjs.host) set_property(TARGET quickjs.host PROPERTY POSITION_INDEPENDENT_CODE ON) diff --git a/samples/apps/forum/src/types/ccf.ts b/samples/apps/forum/src/types/ccf.ts index 6cbb493ccad8..d9501fd59a71 100644 --- a/samples/apps/forum/src/types/ccf.ts +++ b/samples/apps/forum/src/types/ccf.ts @@ -63,7 +63,7 @@ export interface Receipt { nodeId: string; } -export interface State { +export interface HistoricalState { transactionId: string; receipt: Receipt; } @@ -99,7 +99,7 @@ export interface CCF { ): ArrayBuffer; kv: KVMaps; - state?: State; + historicalState?: HistoricalState; } export const ccf = globalThis.ccf as CCF; diff --git a/samples/apps/logging/js/src/logging.js b/samples/apps/logging/js/src/logging.js index a13077e7b163..dfe995b5c7a6 100644 --- a/samples/apps/logging/js/src/logging.js +++ b/samples/apps/logging/js/src/logging.js @@ -35,7 +35,7 @@ export function get_historical(request) { export function get_historical_with_receipt(request) { const result = get_private(request); - result.body.receipt = ccf.state.receipt; + result.body.receipt = ccf.historicalState.receipt; return result; } diff --git a/samples/apps/logging/logging.cpp b/samples/apps/logging/logging.cpp index 65611ea4d15a..548721c72b78 100644 --- a/samples/apps/logging/logging.cpp +++ b/samples/apps/logging/logging.cpp @@ -546,30 +546,8 @@ namespace loggingapp kv::Consensus::View view, kv::Consensus::SeqNo seqno, std::string& error_reason) { - if (consensus == nullptr) - { - error_reason = "Node is not fully configured"; - return false; - } - - const auto tx_view = consensus->get_view(seqno); - const auto committed_seqno = consensus->get_committed_seqno(); - const auto committed_view = consensus->get_view(committed_seqno); - - const auto tx_status = ccf::evaluate_tx_status( - view, seqno, tx_view, committed_view, committed_seqno); - if (tx_status != ccf::TxStatus::Committed) - { - error_reason = fmt::format( - "Only committed transactions can be queried. Transaction {}.{} is " - "{}", - view, - seqno, - ccf::tx_status_to_str(tx_status)); - return false; - } - - return true; + return ccf::historical::is_tx_committed( + consensus, view, seqno, error_reason); }; make_endpoint( "log/private/historical", diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index d1ec9d233ae2..495f88abcab0 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -1070,36 +1070,12 @@ namespace ccfapp if (info.has_value() && info.value().historical) { - // (copied from samples/apps/logging/logging.cpp) auto is_tx_committed = [this]( kv::Consensus::View view, kv::Consensus::SeqNo seqno, std::string& error_reason) { - if (consensus == nullptr) - { - error_reason = "Node is not fully configured"; - return false; - } - - const auto tx_view = consensus->get_view(seqno); - const auto committed_seqno = consensus->get_committed_seqno(); - const auto committed_view = consensus->get_view(committed_seqno); - - const auto tx_status = ccf::evaluate_tx_status( - view, seqno, tx_view, committed_view, committed_seqno); - if (tx_status != ccf::TxStatus::Committed) - { - error_reason = fmt::format( - "Only committed transactions can be queried. Transaction {}.{} " - "is " - "{}", - view, - seqno, - ccf::tx_status_to_str(tx_status)); - return false; - } - - return true; + return ccf::historical::is_tx_committed( + consensus, view, seqno, error_reason); }; ccf::historical::adapter( diff --git a/src/node/historical_queries_adapter.h b/src/node/historical_queries_adapter.h index c9a374f3bcbd..ced0218ce104 100644 --- a/src/node/historical_queries_adapter.h +++ b/src/node/historical_queries_adapter.h @@ -49,6 +49,39 @@ namespace ccf::historical return tx_id_opt; } + static inline bool is_tx_committed( + kv::Consensus* consensus, + kv::Consensus::View view, + kv::Consensus::SeqNo seqno, + std::string& error_reason) + { + if (consensus == nullptr) + { + error_reason = "Node is not fully configured"; + return false; + } + + const auto tx_view = consensus->get_view(seqno); + const auto committed_seqno = consensus->get_committed_seqno(); + const auto committed_view = consensus->get_view(committed_seqno); + + const auto tx_status = ccf::evaluate_tx_status( + view, seqno, tx_view, committed_view, committed_seqno); + if (tx_status != ccf::TxStatus::Committed) + { + error_reason = fmt::format( + "Only committed transactions can be queried. Transaction {}.{} " + "is " + "{}", + view, + seqno, + ccf::tx_status_to_str(tx_status)); + return false; + } + + return true; + }; + #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wunused-function" diff --git a/tests/npm-app/src/types/ccf.ts b/tests/npm-app/src/types/ccf.ts index 6cbb493ccad8..d9501fd59a71 100644 --- a/tests/npm-app/src/types/ccf.ts +++ b/tests/npm-app/src/types/ccf.ts @@ -63,7 +63,7 @@ export interface Receipt { nodeId: string; } -export interface State { +export interface HistoricalState { transactionId: string; receipt: Receipt; } @@ -99,7 +99,7 @@ export interface CCF { ): ArrayBuffer; kv: KVMaps; - state?: State; + historicalState?: HistoricalState; } export const ccf = globalThis.ccf as CCF; From 68add2fcb00dc7b5a74fb3c57f16e60512d2850c Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 15 Mar 2021 07:58:13 +0000 Subject: [PATCH 11/14] fix property name --- src/apps/js_generic/js_generic.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index eb0eae69d074..729b61047365 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -921,7 +921,7 @@ namespace ccfapp JS_SetPropertyStr(ctx, state, "receipt", js_receipt); - JS_SetPropertyStr(ctx, ccf, "state", state); + JS_SetPropertyStr(ctx, ccf, "historicalState", state); } return ccf; From 472e457f982692849174db882b25f9a6e9d0163c Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 15 Mar 2021 09:54:48 +0000 Subject: [PATCH 12/14] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f150f5768b43..cf054684d82a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added -- Single-transaction historical query support has been added to JavaScript endpoints (#2285). +- Historical point query support has been added to JavaScript endpoints (#2285). ## [0.19.0] From 1a020634e289a1566fd39b6bae774cf2b1b51b10 Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 15 Mar 2021 15:12:30 +0000 Subject: [PATCH 13/14] "readonly" -> "mode" --- samples/apps/forum/app.tmpl.json | 24 +++++++++---------- .../apps/forum/tsoa-support/postprocess.js | 8 +++---- samples/apps/logging/js/app.json | 18 +++++++------- src/apps/js_generic/js_generic.cpp | 4 +++- src/node/rpc/endpoint.h | 21 ++++++++++++---- tests/js-app-bundle/app.json | 4 ++-- tests/js-authentication/app.json | 4 ++-- tests/js-content-types/app.json | 8 +++---- tests/js-custom-authorization/app.json | 2 +- tests/js-limits/app.json | 4 ++-- tests/js-modules/basic-module-import/app.json | 2 +- tests/npm-app/app.json | 20 ++++++++-------- 12 files changed, 66 insertions(+), 53 deletions(-) diff --git a/samples/apps/forum/app.tmpl.json b/samples/apps/forum/app.tmpl.json index 499b941a7494..c68070a805a5 100644 --- a/samples/apps/forum/app.tmpl.json +++ b/samples/apps/forum/app.tmpl.json @@ -7,7 +7,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": false + "mode": "readwrite" }, "put": { "js_module": "build/PollControllerProxy.js", @@ -15,7 +15,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": false + "mode": "readwrite" }, "get": { "js_module": "build/PollControllerProxy.js", @@ -23,7 +23,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true + "mode": "readonly" } }, "/polls": { @@ -33,7 +33,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": false + "mode": "readwrite" }, "put": { "js_module": "build/PollControllerProxy.js", @@ -41,7 +41,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": false + "mode": "readwrite" }, "get": { "js_module": "build/PollControllerProxy.js", @@ -49,7 +49,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true + "mode": "readonly" } }, "/site": { @@ -59,7 +59,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true + "mode": "readonly" } }, "/site/polls/create": { @@ -69,7 +69,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true + "mode": "readonly" } }, "/site/opinions/submit": { @@ -79,7 +79,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true + "mode": "readonly" } }, "/site/view": { @@ -89,7 +89,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true + "mode": "readonly" } }, "/csv": { @@ -99,7 +99,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true, + "mode": "readonly", "openapi_merge_patch": { "responses": { "200": { @@ -117,7 +117,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": false, + "mode": "readwrite", "openapi_merge_patch": { "requestBody": { "content": { diff --git a/samples/apps/forum/tsoa-support/postprocess.js b/samples/apps/forum/tsoa-support/postprocess.js index f64602f14a44..2d3112565fe3 100644 --- a/samples/apps/forum/tsoa-support/postprocess.js +++ b/samples/apps/forum/tsoa-support/postprocess.js @@ -4,11 +4,11 @@ import SwaggerParser from "@apidevtools/swagger-parser"; import jsonmergepatch from "json-merge-patch"; // endpoint metadata defaults when first added to endpoints.json -const metadataDefaults = (readonly) => ({ +const metadataDefaults = (mode) => ({ forwarding_required: "always", execute_outside_consensus: "never", authn_policies: ["user_cert"], - readonly: readonly, + mode: mode, }); const distDir = "./dist"; @@ -116,8 +116,8 @@ const oldEndpoints = oldMetadata["endpoints"]; const newEndpoints = newMetadata["endpoints"]; for (const url in newEndpoints) { for (const method in newEndpoints[url]) { - const readonly = method == "get"; - Object.assign(newEndpoints[url][method], metadataDefaults(readonly)); + const mode = method == "get" ? "readonly" : "readwrite"; + Object.assign(newEndpoints[url][method], metadataDefaults(mode)); } } console.log(`Updating ${metadataPath} (if needed)`); diff --git a/samples/apps/logging/js/app.json b/samples/apps/logging/js/app.json index 53e218d5469c..b6872fc0f5ee 100644 --- a/samples/apps/logging/js/app.json +++ b/samples/apps/logging/js/app.json @@ -7,7 +7,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} }, "post": { @@ -16,7 +16,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "readonly": false, + "mode": "readwrite", "openapi": {} }, "delete": { @@ -25,7 +25,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "readonly": false, + "mode": "readwrite", "openapi": {} } }, @@ -36,8 +36,7 @@ "forwarding_required": "never", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "historical": true, - "readonly": true, + "mode": "historical", "openapi": {} } }, @@ -48,8 +47,7 @@ "forwarding_required": "never", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "historical": true, - "readonly": true, + "mode": "historical", "openapi": {} } }, @@ -60,7 +58,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} }, "post": { @@ -69,7 +67,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "readonly": false, + "mode": "readwrite", "openapi": {} }, "delete": { @@ -78,7 +76,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["jwt", "user_cert"], - "readonly": false, + "mode": "readwrite", "openapi": {} } } diff --git a/src/apps/js_generic/js_generic.cpp b/src/apps/js_generic/js_generic.cpp index 729b61047365..dab59e1bbe0a 100644 --- a/src/apps/js_generic/js_generic.cpp +++ b/src/apps/js_generic/js_generic.cpp @@ -1117,7 +1117,9 @@ namespace ccfapp args.tx.ro(ccf::Tables::ENDPOINTS); auto info = endpoints->get(ccf::endpoints::EndpointKey{method, verb}); - if (info.has_value() && info.value().historical) + if ( + info.has_value() && + info.value().mode == ccf::endpoints::Mode::Historical) { auto is_tx_committed = [this]( kv::Consensus::View view, diff --git a/src/node/rpc/endpoint.h b/src/node/rpc/endpoint.h index 68b8dd8c99c1..d21083a617e6 100644 --- a/src/node/rpc/endpoint.h +++ b/src/node/rpc/endpoint.h @@ -39,11 +39,19 @@ namespace ccf Locally, Primary }; + + enum class Mode + { + ReadWrite, + ReadOnly, + Historical + }; } } MSGPACK_ADD_ENUM(ccf::endpoints::ForwardingRequired); MSGPACK_ADD_ENUM(ccf::endpoints::ExecuteOutsideConsensus); +MSGPACK_ADD_ENUM(ccf::endpoints::Mode); namespace ccf { @@ -61,10 +69,17 @@ namespace ccf {ExecuteOutsideConsensus::Locally, "locally"}, {ExecuteOutsideConsensus::Primary, "primary"}}); + DECLARE_JSON_ENUM( + Mode, + {{Mode::ReadWrite, "readwrite"}, + {Mode::ReadOnly, "readonly"}, + {Mode::Historical, "historical"}}); + using AuthnPolicies = std::vector>; struct EndpointProperties { + Mode mode = Mode::ReadWrite; ForwardingRequired forwarding_required = ForwardingRequired::Always; ExecuteOutsideConsensus execute_outside_consensus = ExecuteOutsideConsensus::Never; @@ -73,22 +88,20 @@ namespace ccf nlohmann::json openapi; bool openapi_hidden = false; - bool historical = false; - MSGPACK_DEFINE( forwarding_required, execute_outside_consensus, authn_policies, openapi, openapi_hidden, - historical); + mode); }; DECLARE_JSON_TYPE_WITH_OPTIONAL_FIELDS(EndpointProperties); DECLARE_JSON_REQUIRED_FIELDS( EndpointProperties, forwarding_required, authn_policies); DECLARE_JSON_OPTIONAL_FIELDS( - EndpointProperties, openapi, openapi_hidden, historical); + EndpointProperties, openapi, openapi_hidden, mode); struct EndpointDefinition { diff --git a/tests/js-app-bundle/app.json b/tests/js-app-bundle/app.json index 4781104d8899..7854ddf2e75a 100644 --- a/tests/js-app-bundle/app.json +++ b/tests/js-app-bundle/app.json @@ -7,7 +7,7 @@ "forwarding_required": "never", "execute_outside_consensus": "locally", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": { "requestBody": { "required": true, @@ -80,7 +80,7 @@ "forwarding_required": "never", "execute_outside_consensus": "locally", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": { "parameters": [ { diff --git a/tests/js-authentication/app.json b/tests/js-authentication/app.json index fb8d57aa6e7e..8f04f88899ea 100644 --- a/tests/js-authentication/app.json +++ b/tests/js-authentication/app.json @@ -7,7 +7,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["jwt"], - "readonly": true, + "mode": "readonly", "openapi": {} } }, @@ -25,7 +25,7 @@ "jwt", "no_auth" ], - "readonly": true, + "mode": "readonly", "openapi": {} } } diff --git a/tests/js-content-types/app.json b/tests/js-content-types/app.json index 8becfce67e91..c1c39a1d0122 100644 --- a/tests/js-content-types/app.json +++ b/tests/js-content-types/app.json @@ -7,7 +7,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } }, @@ -18,7 +18,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } }, @@ -29,7 +29,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } }, @@ -40,7 +40,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } } diff --git a/tests/js-custom-authorization/app.json b/tests/js-custom-authorization/app.json index 3b2e68bdba3f..f29768ea0346 100644 --- a/tests/js-custom-authorization/app.json +++ b/tests/js-custom-authorization/app.json @@ -7,7 +7,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } } diff --git a/tests/js-limits/app.json b/tests/js-limits/app.json index 30277f63fd8d..7d5b872fdc5f 100644 --- a/tests/js-limits/app.json +++ b/tests/js-limits/app.json @@ -7,7 +7,7 @@ "forwarding_required": "never", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } }, @@ -18,7 +18,7 @@ "forwarding_required": "never", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } } diff --git a/tests/js-modules/basic-module-import/app.json b/tests/js-modules/basic-module-import/app.json index 04ecc87699e4..1c8248c23b87 100644 --- a/tests/js-modules/basic-module-import/app.json +++ b/tests/js-modules/basic-module-import/app.json @@ -7,7 +7,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": {} } } diff --git a/tests/npm-app/app.json b/tests/npm-app/app.json index 9d66130e67b4..fe24f19667da 100644 --- a/tests/npm-app/app.json +++ b/tests/npm-app/app.json @@ -7,7 +7,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": [], - "readonly": true, + "mode": "readonly", "openapi": { "responses": { "200": { @@ -57,7 +57,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": { "responses": { "200": { @@ -90,7 +90,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": { "requestBody": { "required": true, @@ -125,7 +125,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": { "requestBody": { "required": true, @@ -177,7 +177,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": true, + "mode": "readonly", "openapi": { "requestBody": { "required": true, @@ -218,7 +218,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": false, + "mode": "readonly", "openapi": { "responses": { "200": { @@ -257,7 +257,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": false, + "mode": "readonly", "openapi": { "requestBody": { "required": true, @@ -283,7 +283,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": false, + "mode": "readonly", "openapi": { "responses": { "200": { @@ -313,7 +313,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": false, + "mode": "readwrite", "openapi": { "responses": { "200": { @@ -350,7 +350,7 @@ "forwarding_required": "always", "execute_outside_consensus": "never", "authn_policies": ["user_cert"], - "readonly": false, + "mode": "readonly", "openapi": { "responses": { "200": { From ee9e9a298ebf40329a1a14207bbd612e599a84cd Mon Sep 17 00:00:00 2001 From: Maik Riechert Date: Mon, 15 Mar 2021 15:14:35 +0000 Subject: [PATCH 14/14] update changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf054684d82a..96a9034b986b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Historical point query support has been added to JavaScript endpoints (#2285). +### Changed + +- `"readonly"` has been replaced by `"mode"` in `app.json` in JavaScript apps (#2285). + ## [0.19.0] ### Changed