From ce1d906d3b96b23fa1e5f877f2ebb84b57cd5fb1 Mon Sep 17 00:00:00 2001 From: QuiteStochastic Date: Wed, 27 Apr 2022 17:32:50 +0200 Subject: [PATCH 1/5] explicit proof config --- src/lib/integration_test_cloud_engine/mina_automation.ml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index aaffd48c620..a0cc62866c9 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -185,7 +185,7 @@ module Network_config = struct ; sub_windows_per_window = None ; ledger_depth = None ; work_delay = None - ; block_window_duration_ms = None + ; block_window_duration_ms = Some 150000 ; transaction_capacity = None ; coinbase_amount = None ; supercharged_coinbase_factor = None @@ -209,9 +209,7 @@ module Network_config = struct ; genesis_state_timestamp = Some Core.Time.(to_string_abs ~zone:Zone.utc (now ())) } - ; proof = - None - (* was: Some proof_config; TODO: prebake ledger and only set hash *) + ; proof = Some proof_config (* TODO: prebake ledger and only set hash *) ; ledger = Some { base = Accounts (bp_accounts @ extra_accounts) From 2396d29248c352c0f36cd50d131ef2e26142a238 Mon Sep 17 00:00:00 2001 From: QuiteStochastic Date: Thu, 28 Apr 2022 00:56:06 +0200 Subject: [PATCH 2/5] port over various features in develop to comp --- src/app/test_executive/README.md | 2 +- src/app/test_executive/payments_test.ml | 114 +++++++++++------- .../mina_automation.ml | 6 +- src/lib/integration_test_lib/intf.ml | 2 + src/lib/integration_test_lib/test_config.ml | 8 +- .../integration_test_lib/wait_condition.ml | 16 +++ src/lib/runtime_config/runtime_config.ml | 2 + 7 files changed, 103 insertions(+), 47 deletions(-) diff --git a/src/app/test_executive/README.md b/src/app/test_executive/README.md index 4019b5a3594..987c0383031 100644 --- a/src/app/test_executive/README.md +++ b/src/app/test_executive/README.md @@ -16,7 +16,7 @@ Note: this environment setup assumes that one is a member of o(1) labs and has a ![automated-validation service account "Keys" tab](https://user-images.githubusercontent.com/3465290/112069746-9aaed080-8b29-11eb-83f1-f36876f3ac3d.png) -4) In addition to `GCLOUD_API_KEY` and `GOOGLE_CLOUD_KEYFILE_JSON`, ensure the following other environment variables are also properly set (preferably in in .bashrc or .profile.): +4) In addition to the above mentioned `GCLOUD_API_KEY` and `GOOGLE_CLOUD_KEYFILE_JSON`, ensure the following other environment variables are also properly set (preferably in in .bashrc or .profile.): - `KUBE_CONFIG_PATH`. this should usually be `~/.kube/config` - the following AWS related vars, namely: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION=us-west-2`, - vars relating to ocaml compilation diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index 673d9c93ebe..97edcdf9515 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -49,11 +49,24 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct [ { balance = "1000"; timing = Untimed } ; { balance = "1000"; timing = Untimed } ] - ; num_snark_workers = - 3 - (* this test doesn't need snark workers, however turning it on in this test just to make sure the snark workers function within integration tests *) + ; num_snark_workers = 2 + ; snark_worker_fee = "0.0001" + ; work_delay = Some 1 + ; transaction_capacity = + Some Runtime_config.Proof_keys.Transaction_capacity.small } + (* Call [f] [n] times in sequence *) + let repeat_seq ~n ~f = + let open Malleable_error.Let_syntax in + let rec go n = + if n = 0 then return () + else + let%bind () = f () in + go (n - 1) + in + go n + let run network t = let open Network in let open Malleable_error.Let_syntax in @@ -341,46 +354,63 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct (Wait_condition.signed_command_to_be_included_in_frontier ~txn_hash:hash ~node_included_in:(`Node timed_node_c))) in - section "unable to send payment from timed account using illiquid tokens" - (let amount = Currency.Amount.of_int 25_000_000_000_000 in - let receiver = untimed_node_b in + let%bind () = + section "unable to send payment from timed account using illiquid tokens" + (let amount = Currency.Amount.of_int 25_000_000_000_000 in + let receiver = untimed_node_b in + let%bind receiver_pub_key = Util.pub_key_of_node receiver in + let sender = timed_node_c in + let%bind sender_pub_key = Util.pub_key_of_node sender in + let%bind { total_balance = timed_node_c_total; _ } = + Network.Node.must_get_account_data ~logger timed_node_c + ~public_key:sender_pub_key + in + [%log info] "timed_node_c total balance: %s" + (Currency.Balance.to_formatted_string timed_node_c_total) ; + [%log info] + "Attempting to send txn from timed_node_c to untimed_node_a for \ + amount of %s" + (Currency.Amount.to_formatted_string amount) ; + (* TODO: refactor this using new [expect] dsl when it's available *) + let open Deferred.Let_syntax in + match%bind + Node.send_payment ~logger sender ~sender_pub_key ~receiver_pub_key + ~amount ~fee + with + | Ok _ -> + Malleable_error.soft_error_string ~value:() + "Payment succeeded, but expected it to fail because of a \ + minimum balance violation" + | Error error -> + (* expect GraphQL error due to insufficient funds *) + let err_str = Error.to_string_mach error in + let err_str_lowercase = String.lowercase err_str in + if + String.is_substring ~substring:"insufficient_funds" + err_str_lowercase + then ( + [%log info] "Got expected insufficient funds error from GraphQL" ; + Malleable_error.return () ) + else ( + [%log error] + "Payment failed in GraphQL, but for unexpected reason: %s" + err_str ; + Malleable_error.soft_error_format ~value:() + "Payment failed for unexpected reason: %s" err_str )) + in + section + "send out a bunch more txns to fill up the snark ledger, then wait for \ + proofs to be emitted" + (let receiver = untimed_node_a in let%bind receiver_pub_key = Util.pub_key_of_node receiver in - let sender = timed_node_c in + let sender = untimed_node_b in let%bind sender_pub_key = Util.pub_key_of_node sender in - let%bind { total_balance = timed_node_c_total; _ } = - Network.Node.must_get_account_data ~logger timed_node_c - ~public_key:sender_pub_key + let%bind () = + repeat_seq ~n:10 ~f:(fun () -> + Network.Node.must_send_payment ~logger sender ~sender_pub_key + ~receiver_pub_key ~amount:Currency.Amount.one ~fee + >>| ignore) in - [%log info] "timed_node_c total balance: %s" - (Currency.Balance.to_formatted_string timed_node_c_total) ; - [%log info] - "Attempting to send txn from timed_node_c to untimed_node_a for \ - amount of %s" - (Currency.Amount.to_formatted_string amount) ; - (* TODO: refactor this using new [expect] dsl when it's available *) - let open Deferred.Let_syntax in - match%bind - Node.send_payment ~logger sender ~sender_pub_key ~receiver_pub_key - ~amount ~fee - with - | Ok _ -> - Malleable_error.soft_error_string ~value:() - "Payment succeeded, but expected it to fail because of a minimum \ - balance violation" - | Error error -> - (* expect GraphQL error due to insufficient funds *) - let err_str = Error.to_string_mach error in - let err_str_lowercase = String.lowercase err_str in - if - String.is_substring ~substring:"insufficient_funds" - err_str_lowercase - then ( - [%log info] "Got expected insufficient funds error from GraphQL" ; - Malleable_error.return () ) - else ( - [%log error] - "Payment failed in GraphQL, but for unexpected reason: %s" - err_str ; - Malleable_error.soft_error_format ~value:() - "Payment failed for unexpected reason: %s" err_str )) + wait_for t + (Wait_condition.ledger_proofs_emitted_since_genesis ~num_proofs:1)) end diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index a0cc62866c9..98f75b8d497 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -91,6 +91,8 @@ module Network_config = struct ; log_precomputed_blocks ; snark_worker_fee ; snark_worker_public_key (* ; aux_account_balance *) + ; work_delay + ; transaction_capacity } = test_config in @@ -184,9 +186,9 @@ module Network_config = struct { Runtime_config.Proof_keys.level = Some proof_level ; sub_windows_per_window = None ; ledger_depth = None - ; work_delay = None + ; work_delay ; block_window_duration_ms = Some 150000 - ; transaction_capacity = None + ; transaction_capacity ; coinbase_amount = None ; supercharged_coinbase_factor = None ; account_creation_fee = None diff --git a/src/lib/integration_test_lib/intf.ml b/src/lib/integration_test_lib/intf.ml index 002a79605f3..bfb2e05a047 100644 --- a/src/lib/integration_test_lib/intf.ml +++ b/src/lib/integration_test_lib/intf.ml @@ -338,6 +338,8 @@ module Dsl = struct txn_hash:Transaction_hash.t -> node_included_in:[ `Any_node | `Node of Engine.Network.Node.t ] -> t + + val ledger_proofs_emitted_since_genesis : num_proofs:int -> t end module type Util_intf = sig diff --git a/src/lib/integration_test_lib/test_config.ml b/src/lib/integration_test_lib/test_config.ml index 17c079834e8..36938b5ee92 100644 --- a/src/lib/integration_test_lib/test_config.ml +++ b/src/lib/integration_test_lib/test_config.ml @@ -33,7 +33,10 @@ type t = ; num_archive_nodes : int ; log_precomputed_blocks : bool ; snark_worker_fee : string - ; snark_worker_public_key : string (* ; aux_account_balance : string option *) + ; snark_worker_public_key : string + ; work_delay : int option + ; transaction_capacity : + Runtime_config.Proof_keys.Transaction_capacity.t option } let default = @@ -53,5 +56,6 @@ let default = ; snark_worker_public_key = (let pk, _ = (Lazy.force Mina_base.Sample_keypairs.keypairs).(0) in Signature_lib.Public_key.Compressed.to_string pk) - (* ; aux_account_balance = None *) + ; work_delay = None + ; transaction_capacity = None } diff --git a/src/lib/integration_test_lib/wait_condition.ml b/src/lib/integration_test_lib/wait_condition.ml index 6b7459d39b6..50171d84c5c 100644 --- a/src/lib/integration_test_lib/wait_condition.ml +++ b/src/lib/integration_test_lib/wait_condition.ml @@ -150,4 +150,20 @@ struct ; soft_timeout = Slots soft_timeout_in_slots ; hard_timeout = Slots (soft_timeout_in_slots * 2) } + + let ledger_proofs_emitted_since_genesis ~num_proofs = + let open Network_state in + let check () (state : Network_state.t) = + if state.snarked_ledgers_generated >= num_proofs then Predicate_passed + else Predicate_continuation () + in + let description = + Printf.sprintf "[%d] snarked_ledgers to be generated since genesis" + num_proofs + in + { description + ; predicate = Network_state_predicate (check (), check) + ; soft_timeout = Slots 15 + ; hard_timeout = Slots 20 + } end diff --git a/src/lib/runtime_config/runtime_config.ml b/src/lib/runtime_config/runtime_config.ml index d3d01658b41..382fcf0a5b9 100644 --- a/src/lib/runtime_config/runtime_config.ml +++ b/src/lib/runtime_config/runtime_config.ml @@ -643,6 +643,8 @@ module Proof_keys = struct let of_yojson json = Result.bind ~f:of_json_layout (Json_layout.Proof_keys.Transaction_capacity.of_yojson json) + + let small : t = Log_2 2 end type t = From b5efd6cb2ce5b293e5ec8b9577a9544d56c77139 Mon Sep 17 00:00:00 2001 From: QuiteStochastic Date: Thu, 28 Apr 2022 11:14:35 +0200 Subject: [PATCH 3/5] drop slot time to 2 minutes --- src/app/test_executive/payments_test.ml | 3 ++- src/lib/integration_test_cloud_engine/mina_automation.ml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index 97edcdf9515..0a61a0b5b90 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -49,7 +49,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct [ { balance = "1000"; timing = Untimed } ; { balance = "1000"; timing = Untimed } ] - ; num_snark_workers = 2 + ; num_snark_workers = 4 ; snark_worker_fee = "0.0001" ; work_delay = Some 1 ; transaction_capacity = @@ -406,6 +406,7 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let sender = untimed_node_b in let%bind sender_pub_key = Util.pub_key_of_node sender in let%bind () = + (* deepthi says that to fill up a `small` transaction capacity with work delay of 1, there needs to be 12 total txns sent. 2 successfull txn are sent in the prior course of this test, so spamming out at least 10 more here will trigger a ledger proof to be emitted *) repeat_seq ~n:10 ~f:(fun () -> Network.Node.must_send_payment ~logger sender ~sender_pub_key ~receiver_pub_key ~amount:Currency.Amount.one ~fee diff --git a/src/lib/integration_test_cloud_engine/mina_automation.ml b/src/lib/integration_test_cloud_engine/mina_automation.ml index 98f75b8d497..29ebd6da858 100644 --- a/src/lib/integration_test_cloud_engine/mina_automation.ml +++ b/src/lib/integration_test_cloud_engine/mina_automation.ml @@ -187,7 +187,7 @@ module Network_config = struct ; sub_windows_per_window = None ; ledger_depth = None ; work_delay - ; block_window_duration_ms = Some 150000 + ; block_window_duration_ms = Some 120000 ; transaction_capacity ; coinbase_amount = None ; supercharged_coinbase_factor = None From 335a5b29d07197ba5c0961ebdc2ad33b3e90f782 Mon Sep 17 00:00:00 2001 From: QuiteStochastic Date: Thu, 28 Apr 2022 11:30:00 +0200 Subject: [PATCH 4/5] update the readme with a better link for extensions --- README-dev.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README-dev.md b/README-dev.md index 3a6af4b7f72..a7843a9cadb 100644 --- a/README-dev.md +++ b/README-dev.md @@ -85,7 +85,7 @@ let g:syntastic_ocaml_checkers=['merlin'] - If you use vscode, you might like these extensions - [OCaml and Reason IDE](https://marketplace.visualstudio.com/items?itemName=freebroccolo.reasonml) - - [Dune](https://marketplace.visualstudio.com/items?itemName=maelvalais.dune) + - [OCaml Platform](https://marketplace.visualstudio.com/items?itemName=ocamllabs.ocaml-platform) - If you use emacs, besides the `opam` packages mentioned above, also install `tuareg`, and add the following to your .emacs file: From 09b0ee18b8ff2e68ef6b8e87ddae74334fa3c29e Mon Sep 17 00:00:00 2001 From: QuiteStochastic Date: Fri, 29 Apr 2022 02:49:11 +0200 Subject: [PATCH 5/5] adding informative comment, ty deepthi --- src/app/test_executive/payments_test.ml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/app/test_executive/payments_test.ml b/src/app/test_executive/payments_test.ml index 0a61a0b5b90..ac7fabeae68 100644 --- a/src/app/test_executive/payments_test.ml +++ b/src/app/test_executive/payments_test.ml @@ -406,7 +406,21 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct let sender = untimed_node_b in let%bind sender_pub_key = Util.pub_key_of_node sender in let%bind () = - (* deepthi says that to fill up a `small` transaction capacity with work delay of 1, there needs to be 12 total txns sent. 2 successfull txn are sent in the prior course of this test, so spamming out at least 10 more here will trigger a ledger proof to be emitted *) + (* + To fill up a `small` transaction capacity with work delay of 1, + there needs to be 12 total txns sent. + + Calculation is as follows: + Max number trees in the scan state is + `(transaction_capacity_log+1) * (work_delay+1)` + and for 2^2 transaction capacity and work delay 1 it is + `(2+1)*(1+1)=6`. + Per block there can be 2 transactions included (other two slots would be for a coinbase and fee transfers). + In the initial state of the network, the scan state waits till all the trees are filled before emitting a proof from the first tree. + Hence, 6*2 = 12 transactions untill we get the first snarked ledger. + + 2 successful txn are sent in the prior course of this test, + so spamming out at least 10 more here will trigger a ledger proof to be emitted *) repeat_seq ~n:10 ~f:(fun () -> Network.Node.must_send_payment ~logger sender ~sender_pub_key ~receiver_pub_key ~amount:Currency.Amount.one ~fee