port over various features in develop to comp

src/app/test_executive/README.md

@@ -16,7 +16,7 @@ Note: this environment setup assumes that one is a member of o(1) labs and has a
 
 ![automated-validation service account "Keys" tab](https://user-images.githubusercontent.com/3465290/112069746-9aaed080-8b29-11eb-83f1-f36876f3ac3d.png)
 
-4) In addition to `GCLOUD_API_KEY` and `GOOGLE_CLOUD_KEYFILE_JSON`, ensure the following other environment variables are also properly set (preferably in in .bashrc or .profile.):
+4) In addition to the above mentioned `GCLOUD_API_KEY` and `GOOGLE_CLOUD_KEYFILE_JSON`, ensure the following other environment variables are also properly set (preferably in in .bashrc or .profile.):
 - `KUBE_CONFIG_PATH`.  this should usually be `~/.kube/config`
 - the following AWS related vars, namely: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_DEFAULT_REGION=us-west-2`,
 - vars relating to ocaml compilation

src/app/test_executive/payments_test.ml

@@ -49,11 +49,24 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct
         [ { balance = "1000"; timing = Untimed }
         ; { balance = "1000"; timing = Untimed }
         ]
-    ; num_snark_workers =
-        3
-        (* this test doesn't need snark workers, however turning it on in this test just to make sure the snark workers function within integration tests *)
+    ; num_snark_workers = 2
+    ; snark_worker_fee = "0.0001"
+    ; work_delay = Some 1
+    ; transaction_capacity =
+        Some Runtime_config.Proof_keys.Transaction_capacity.small
     }
 
+  (* Call [f] [n] times in sequence *)
+  let repeat_seq ~n ~f =
+    let open Malleable_error.Let_syntax in
+    let rec go n =
+      if n = 0 then return ()
+      else
+        let%bind () = f () in
+        go (n - 1)
+    in
+    go n
+
   let run network t =
     let open Network in
     let open Malleable_error.Let_syntax in
@@ -341,46 +354,63 @@ module Make (Inputs : Intf.Test.Inputs_intf) = struct
            (Wait_condition.signed_command_to_be_included_in_frontier
               ~txn_hash:hash ~node_included_in:(`Node timed_node_c)))
     in
-    section "unable to send payment from timed account using illiquid tokens"
-      (let amount = Currency.Amount.of_int 25_000_000_000_000 in
-       let receiver = untimed_node_b in
+    let%bind () =
+      section "unable to send payment from timed account using illiquid tokens"
+        (let amount = Currency.Amount.of_int 25_000_000_000_000 in
+         let receiver = untimed_node_b in
+         let%bind receiver_pub_key = Util.pub_key_of_node receiver in
+         let sender = timed_node_c in
+         let%bind sender_pub_key = Util.pub_key_of_node sender in
+         let%bind { total_balance = timed_node_c_total; _ } =
+           Network.Node.must_get_account_data ~logger timed_node_c
+             ~public_key:sender_pub_key
+         in
+         [%log info] "timed_node_c total balance: %s"
+           (Currency.Balance.to_formatted_string timed_node_c_total) ;
+         [%log info]
+           "Attempting to send txn from timed_node_c to untimed_node_a for \
+            amount of %s"
+           (Currency.Amount.to_formatted_string amount) ;
+         (* TODO: refactor this using new [expect] dsl when it's available *)
+         let open Deferred.Let_syntax in
+         match%bind
+           Node.send_payment ~logger sender ~sender_pub_key ~receiver_pub_key
+             ~amount ~fee
+         with
+         | Ok _ ->
+             Malleable_error.soft_error_string ~value:()
+               "Payment succeeded, but expected it to fail because of a \
+                minimum balance violation"
+         | Error error ->
+             (* expect GraphQL error due to insufficient funds *)
+             let err_str = Error.to_string_mach error in
+             let err_str_lowercase = String.lowercase err_str in
+             if
+               String.is_substring ~substring:"insufficient_funds"
+                 err_str_lowercase
+             then (
+               [%log info] "Got expected insufficient funds error from GraphQL" ;
+               Malleable_error.return () )
+             else (
+               [%log error]
+                 "Payment failed in GraphQL, but for unexpected reason: %s"
+                 err_str ;
+               Malleable_error.soft_error_format ~value:()
+                 "Payment failed for unexpected reason: %s" err_str ))
+    in
+    section
+      "send out a bunch more txns to fill up the snark ledger, then wait for \
+       proofs to be emitted"
+      (let receiver = untimed_node_a in
        let%bind receiver_pub_key = Util.pub_key_of_node receiver in
-       let sender = timed_node_c in
+       let sender = untimed_node_b in
        let%bind sender_pub_key = Util.pub_key_of_node sender in
-       let%bind { total_balance = timed_node_c_total; _ } =
-         Network.Node.must_get_account_data ~logger timed_node_c
-           ~public_key:sender_pub_key
+       let%bind () =
+         repeat_seq ~n:10 ~f:(fun () ->
+             Network.Node.must_send_payment ~logger sender ~sender_pub_key
+               ~receiver_pub_key ~amount:Currency.Amount.one ~fee
+             >>| ignore)
        in
-       [%log info] "timed_node_c total balance: %s"
-         (Currency.Balance.to_formatted_string timed_node_c_total) ;
-       [%log info]
-         "Attempting to send txn from timed_node_c to untimed_node_a for \
-          amount of %s"
-         (Currency.Amount.to_formatted_string amount) ;
-       (* TODO: refactor this using new [expect] dsl when it's available *)
-       let open Deferred.Let_syntax in
-       match%bind
-         Node.send_payment ~logger sender ~sender_pub_key ~receiver_pub_key
-           ~amount ~fee
-       with
-       | Ok _ ->
-           Malleable_error.soft_error_string ~value:()
-             "Payment succeeded, but expected it to fail because of a minimum \
-              balance violation"
-       | Error error ->
-           (* expect GraphQL error due to insufficient funds *)
-           let err_str = Error.to_string_mach error in
-           let err_str_lowercase = String.lowercase err_str in
-           if
-             String.is_substring ~substring:"insufficient_funds"
-               err_str_lowercase
-           then (
-             [%log info] "Got expected insufficient funds error from GraphQL" ;
-             Malleable_error.return () )
-           else (
-             [%log error]
-               "Payment failed in GraphQL, but for unexpected reason: %s"
-               err_str ;
-             Malleable_error.soft_error_format ~value:()
-               "Payment failed for unexpected reason: %s" err_str ))
+       wait_for t
+         (Wait_condition.ledger_proofs_emitted_since_genesis ~num_proofs:1))
 end

src/lib/integration_test_cloud_engine/mina_automation.ml

@@ -91,6 +91,8 @@ module Network_config = struct
         ; log_precomputed_blocks
         ; snark_worker_fee
         ; snark_worker_public_key (* ; aux_account_balance *)
+        ; work_delay
+        ; transaction_capacity
         } =
       test_config
     in
@@ -184,9 +186,9 @@ module Network_config = struct
       { Runtime_config.Proof_keys.level = Some proof_level
       ; sub_windows_per_window = None
       ; ledger_depth = None
-      ; work_delay = None
+      ; work_delay
       ; block_window_duration_ms = Some 150000
-      ; transaction_capacity = None
+      ; transaction_capacity
       ; coinbase_amount = None
       ; supercharged_coinbase_factor = None
       ; account_creation_fee = None

src/lib/integration_test_lib/intf.ml

@@ -338,6 +338,8 @@ module Dsl = struct
          txn_hash:Transaction_hash.t
       -> node_included_in:[ `Any_node | `Node of Engine.Network.Node.t ]
       -> t
+
+    val ledger_proofs_emitted_since_genesis : num_proofs:int -> t
   end
 
   module type Util_intf = sig

src/lib/integration_test_lib/test_config.ml

@@ -33,7 +33,10 @@ type t =
   ; num_archive_nodes : int
   ; log_precomputed_blocks : bool
   ; snark_worker_fee : string
-  ; snark_worker_public_key : string (* ; aux_account_balance : string option *)
+  ; snark_worker_public_key : string
+  ; work_delay : int option
+  ; transaction_capacity :
+      Runtime_config.Proof_keys.Transaction_capacity.t option
   }
 
 let default =
@@ -53,5 +56,6 @@ let default =
   ; snark_worker_public_key =
       (let pk, _ = (Lazy.force Mina_base.Sample_keypairs.keypairs).(0) in
        Signature_lib.Public_key.Compressed.to_string pk)
-      (* ; aux_account_balance = None *)
+  ; work_delay = None
+  ; transaction_capacity = None
   }

src/lib/integration_test_lib/wait_condition.ml

@@ -150,4 +150,20 @@ struct
     ; soft_timeout = Slots soft_timeout_in_slots
     ; hard_timeout = Slots (soft_timeout_in_slots * 2)
     }
+
+  let ledger_proofs_emitted_since_genesis ~num_proofs =
+    let open Network_state in
+    let check () (state : Network_state.t) =
+      if state.snarked_ledgers_generated >= num_proofs then Predicate_passed
+      else Predicate_continuation ()
+    in
+    let description =
+      Printf.sprintf "[%d] snarked_ledgers to be generated since genesis"
+        num_proofs
+    in
+    { description
+    ; predicate = Network_state_predicate (check (), check)
+    ; soft_timeout = Slots 15
+    ; hard_timeout = Slots 20
+    }
 end

src/lib/runtime_config/runtime_config.ml

@@ -643,6 +643,8 @@ module Proof_keys = struct
     let of_yojson json =
       Result.bind ~f:of_json_layout
         (Json_layout.Proof_keys.Transaction_capacity.of_yojson json)
+
+    let small : t = Log_2 2
   end
 
   type t =