workbench: fix Shelley genesis for Alonzo+ & some profile refactoring

nix/workbench/lib.jq

@@ -11,3 +11,6 @@ def color:
 
 def colorly($col; $x):
   "\(color[$col])\($x)\(color["off"])";
+
+def drop_nulls:
+  map(select(. != null));

nix/workbench/lib.sh

@@ -73,6 +73,11 @@ progress() {
     msg "$(with_color green $subsys):  $(with_color blue $*)"
 }
 
+progress_ne() {
+    local subsys=$1; shift
+    msg_ne "$(with_color green $subsys):  $(with_color blue $*)"
+}
+
 fail() {
     msg "$(with_color red $*)"
     exit 1

nix/workbench/profile.sh

@@ -3,7 +3,7 @@ usage_profile() {
     list                  List profile names (json)
     all-profiles | all    All profile contents (json)
     compose NAME..        Create a profile composed from named profiles
-    json-by-name NAME     Get contents of either named profile, or profile JSON desc
+    json NAME             Get contents of either named profile, or profile JSON desc
     describe NAME         Print a human description of a profile
     node-specs PROFILE-NAME/JSON
                           Print node specs JSON for the given profile and environment
@@ -20,7 +20,7 @@ global_profile_eras=(
 )
 
 profile() {
-local op=${1:-list}; test $# -gt 0 && shift
+local op=${1:-show}; test $# -gt 0 && shift
 
 case "$op" in
     list | names )
@@ -60,9 +60,9 @@ case "$op" in
           | add
           ';;
 
-    json-by-name )
-        local usage="USAGE: wb profile $op NAME"
-        local name=${1:?$usage}
+    json | show )
+        local usage="USAGE: wb profile $op [NAME=<current-shell-profile>"
+        local name=${1:-${WORKBENCH_SHELL_PROFILE:?variable unset, no profile name to use as a default.}}
 
         if test -f  "$name"
         then jq '.' "$name"
@@ -74,7 +74,7 @@ case "$op" in
         local usage="USAGE: wb profile $op NAME"
         local name=${1:?$usage}
 
-        profile json-by-name $name |
+        profile json $name |
         jq 'include "derived";
 
            profile_pretty_describe(.)
@@ -83,7 +83,7 @@ case "$op" in
     has-preset )
         local usage="USAGE: wb profile $op NAME"
         local profile=${1:?$usage}
-        profile json-by-name "$profile" | jqtest ".preset != null";;
+        profile json "$profile" | jqtest ".preset != null";;
 
     preset-get-file )
         local usage="USAGE: wb profile $op PRESET-NAME DESC FILE"
@@ -108,7 +108,7 @@ case "$op" in
         )
         ## WARNING: this is structured in correspondence
         ##          with the output generated by cardano-topology
-        profile json-by-name "$profile" | jq '
+        profile json "$profile" | jq '
           include "profile-run";
           . as $prof
           | profile_node_specs($env; $prof)
@@ -118,7 +118,7 @@ case "$op" in
         local usage="USAGE: wb profile $op PROFILE-JSON"
         local profile=${1:?$usage}
 
-        local offset=$(profile json-by-name "$profile" |
+        local offset=$(profile json "$profile" |
                        jq '.derived.genesis_future_offset' --raw-output)
         local start=$(date '+%s' --date="now + $offset")
         local args=(
@@ -129,7 +129,7 @@ case "$op" in
             --arg 'start_tag'       "$(date --date=@$start --utc +'%Y'-'%m'-'%d'-'%H.%M')"
             --arg 'systemStart'     "$(date --date=@$start --utc --iso-8601=s | cut -c-19)Z"
         )
-        profile json-by-name "$profile" | jq '
+        profile json "$profile" | jq '
           include "profile-run";
 
           . as $prof

nix/workbench/profiles/default.nix

@@ -10,7 +10,7 @@
 
 let
   baseJSON = runWorkbenchJqOnly "profile-${profileName}.json"
-    "profile json-by-name ${profileName}";
+    "profile json ${profileName}";
   JSON =
     if profileOverride == {}
     then baseJSON

nix/workbench/profiles/defaults.jq

@@ -102,6 +102,15 @@ def era_defaults($era):
   }
 
 , alonzo:
-  {
+  { genesis:
+    { shelley:
+      { protocolParams:
+        { protocolVersion:
+          { major: 5
+          , minor: 0
+          }
+        }
+      }
+    }
   }
 } | (.common * .[$era]);

nix/workbench/profiles/derived.jq

@@ -86,64 +86,76 @@ def add_derived_params:
    then $shutdown_slots / $gsis.epoch_length | ceil
    else $gtor.epochs
    end)                                      as $effective_epochs
-| ($epoch_duration * $effective_epochs)      as $generator_duration
+| ($epoch_duration * $effective_epochs)      as $generator_requested_duration
 | ($shutdown_slots | may_mult($gsis.slot_duration)) as $shutdown_time
+| ([ $generator_requested_duration
+   , $shutdown_time
+   ] | drop_nulls | min)                     as $generator_duration
 
 ## Tx count for inferred absolute duration.
 ##   Note that this the workload would take longer, if we saturate the cluster.
 | ($gtor.tx_count // ($generator_duration * $gtor.tps))
                                              as $generator_tx_count
-
 ## Effective cluster composition:
 | (if $compo.dense_pool_density > 1
    then { singular:  $compo.n_singular_hosts
         , dense:     $compo.n_dense_hosts }
    else { singular: ($compo.n_singular_hosts + $compo.n_dense_hosts)
         , dense:     0 }
    end)                                      as $hosts
-| $hosts.singular                            as $n_singular_pools
 | ($hosts.dense * $compo.dense_pool_density) as $n_dense_pools
-| ($n_singular_pools + $n_dense_pools)       as $n_pools
+| ($hosts.singular + $n_dense_pools)         as $n_pools
 
 | ($gsis.delegators // $n_pools)             as $effective_delegators
 
-## Stuffed UTxO is what we need over requested-UTxO + delegators' UTxO:
-| $effective_delegators                      as $utxo_delegated
 | ($generator_tx_count * $gtor.inputs_per_tx)
                                              as $utxo_generated
-| ([ $gsis.utxo - $utxo_generated - $effective_delegators
-   , 0
-   ] | max)                                  as $utxo_stuffed
 | (($gsis.max_block_size / default_value_tx_size_estimate) | floor)
                                              as $default_value_tx_per_block_estimate
-| ($generator_tx_count / $default_value_tx_per_block_estimate | . * 1.15 | ceil)
-                                             as $generator_blocks_lower_bound
 ## Note how derivations come in phases, too:
 ##
 | (## First derivation:
    { common:
+     (($gsis.per_pool_balance * $n_pools) as $supply_delegated
+     |
      { derived:
-         { utxo_delegated:                $utxo_delegated
+         { supply_delegated:              $supply_delegated
+         , supply_total:                  ($supply_delegated + $gsis.funds_balance)
+
+         , utxo_delegated:                $effective_delegators
          , utxo_generated:                $utxo_generated
-         , utxo_stuffed:                  $utxo_stuffed
+           ## Stuffed UTxO is what we need over requested-UTxO + delegators' UTxO:
+         , utxo_stuffed:                  ([ $gsis.utxo - $utxo_generated - $effective_delegators
+                                           , 0
+                                           ] | max)
+
+         , delegators_effective:          ([ $n_pools
+                                           , $gsis.delegators
+                                           ] | max)
+
          , dataset_measure:               $dataset_measure
          , dataset_induced_startup_delay_optimistic:   $dataset_induced_startup_delay_optimistic
          , dataset_induced_startup_delay_conservative: $dataset_induced_startup_delay_conservative
+
          , genesis_future_offset:         $genesis_future_offset
          , epoch_duration:                $epoch_duration
-         , effective_epochs:              $effective_epochs
          , generator_duration:            $generator_duration
+         , shutdown_time:                 $shutdown_time
+
+         , effective_epochs:              $effective_epochs
+
          , generator_tx_count:            $generator_tx_count
+
          , default_value_tx_size_estimate: default_value_tx_size_estimate
          , default_value_tx_per_block_estimate: $default_value_tx_per_block_estimate
-         , generator_blocks_lower_bound:  $generator_blocks_lower_bound
-         , shutdown_time:                 $shutdown_time
+         , generator_blocks_lower_bound:  ($generator_tx_count / $default_value_tx_per_block_estimate
+                                           | . * 1.15 | ceil)
          }
      , composition:
          { n_hosts:               ($compo.n_bft_hosts + $hosts.singular + $hosts.dense)
          , n_pools:               $n_pools
          , n_singular_hosts:      $hosts.singular
-         , n_singular_pools:      $n_singular_pools
+         , n_singular_pools:      $hosts.singular
          , n_dense_hosts:         $hosts.dense
          , n_dense_pools:         $n_dense_pools
          , n_pool_hosts:          ($hosts.singular + $hosts.dense)
@@ -177,7 +189,7 @@ def add_derived_params:
          { minimum_chain_density:      ($gsis.active_slots_coeff * 0.5)
          , cluster_startup_overhead_s: $dataset_induced_startup_delay_conservative
          }
-     }
+     })
    }
    | . *
    ## Second derivation:

nix/workbench/profiles/genesis.jq

@@ -1,32 +1,28 @@
 ## WARNING:  keep in sync with 'profile-cache-key-input' below this one: vvv
 ##
 def profile_cli_args($p):
-($p.genesis.per_pool_balance * $p.composition.n_pools) as $pools_balance
-|
 { common:
   { createSpec:
-    [ "--supply",                  ($pools_balance + $p.genesis.funds_balance)
+    [ "--supply",                  $p.derived.supply_total
     , "--testnet-magic",           $p.genesis.network_magic
     , "--gen-genesis-keys",        $p.composition.n_bft_hosts
     , "--gen-utxo-keys",           1
     ]
  , createFinalIncremental:
-    ([ "--supply",                 ($p.genesis.funds_balance)
+    ([ "--supply",                 $p.genesis.funds_balance
      , "--gen-utxo-keys",          1
      ] +
      if $p.composition.dense_pool_density != 1
      then
      [  ]
      else [] end)
   , createFinalBulk:
-    ([ "--supply",                 ($p.genesis.funds_balance)
+    ([ "--supply",                 $p.genesis.funds_balance
      , "--gen-utxo-keys",          1
      , "--gen-genesis-keys",       $p.composition.n_bft_hosts
-     , "--supply-delegated",       $pools_balance
+     , "--supply-delegated",       $p.derived.supply_delegated
      , "--gen-pools",              $p.composition.n_pools
-     , "--gen-stake-delegs",       ([ $p.composition.n_pools
-                                    , $p.genesis.delegators ]
-                                     | max)
+     , "--gen-stake-delegs",       $p.derived.delegators_effective
      , "--testnet-magic",          $p.genesis.network_magic
      , "--num-stuffed-utxo",       $p.derived.utxo_stuffed
      ] +

nix/workbench/profiles/variants.jq

@@ -3,23 +3,41 @@
 
 def genesis_profile_variants:
 
-  { scenario: "fixed-loaded"
-  , composition:
-    { n_singular_hosts:               2
-    , n_dense_hosts:                  0
-    }
-  , genesis:
-    { utxo:                           6000000
-    , delegators:                     1300000
-    , max_block_size:                 80000
-    , epoch_length:                   600
-    , parameter_k:                    3
-    }
-  , node:
-    { shutdown_on_slot_synced: 2400
-    }
-  , generator: { tps: 15 }
-  } as $forge_stress_base
+    { scenario: "fixed-loaded"
+    , composition:
+      { n_singular_hosts:               2
+      , n_dense_hosts:                  0
+      }
+    , genesis:
+      { utxo:                           6000000
+      , delegators:                     1300000
+      , max_block_size:                 80000
+      , epoch_length:                   600
+      , parameter_k:                    3
+      }
+    , node:
+      { shutdown_on_slot_synced: 2400
+      }
+    , generator: { tps: 15 }
+    } as $forge_stress_base
+
+  | { genesis:
+      { alonzo:
+        { maxTxExUnits:
+          { exUnitsMem:                 12500000
+          }
+        }
+      }
+    , generator:
+      { inputs_per_tx:                  1
+      , outputs_per_tx:                 1
+      , plutusMode:                     true
+      , plutusAutoMode:                 true
+      }
+    , analysis:
+      { filters:                        ["base", "size-small"]
+      }
+    } as $plutus_base
   |
 
   ## Baseline:
@@ -79,34 +97,27 @@ def genesis_profile_variants:
     , generator: { tps: 10 }
     }
 
-  , { name: "forge-stress"
-    } * $forge_stress_base
+  , $forge_stress_base *
+    { name: "forge-stress"
+    }
 
-  , { name: "forge-stress-newtracing"
+  , $forge_stress_base *
+    { name: "forge-stress-newtracing"
     , node:
       { tracing_backend:                "trace-dispatcher"
       }
-    } * $forge_stress_base
+    }
 
-  , { name: "forge-stress-plutus"
+  , $forge_stress_base *
+    $plutus_base *
+    { name: "forge-stress-plutus"
     , composition:
       { n_singular_hosts:               1
       }
-    , genesis:
-      { alonzo:
-        { maxTxExUnits:
-          { exUnitsMem:                 12500000
-          }
-        }
-      }
     , generator:
-      { inputs_per_tx:                  1
-      , outputs_per_tx:                 1
-      , tx_count:                       800
-      , plutusMode:                     true
-      , plutusAutoMode:                 true
+      { tx_count:                       800
       }
-    } * $forge_stress_base
+    }
 
   , { name: "quick"
     , scenario: "fixed-loaded"

nix/workbench/run.sh

@@ -302,7 +302,7 @@ case "$op" in
         else
             profile has-profile          "$profile_name" ||
                 fatal  "no such profile:  $profile_name"
-            profile json-by-name         "$profile_name" > "$dir"/profile.json
+            profile json                 "$profile_name" > "$dir"/profile.json
         fi
         jq '.' <<<$node_specs > "$dir"/node-specs.json
 
@@ -464,12 +464,12 @@ EOF
     start )
         local usage="USAGE: wb run $op [--idle] [--scenario NAME] [--analyse] TAG"
 
-        local scenario_override= analyse= analysis_can_fail=
+        local scenario_override= analyse=yes analysis_can_fail=
         while test $# -gt 0
         do case "$1" in
                --idle )              scenario_override='generic-idle';;
                --scenario | -s )     scenario_override=$2; shift;;
-               --analyse | -a )      analyse=t;;
+               --no-analysis )       analyse=;;
                --analysis-can-fail ) analysis_can_fail=t;;
                --* ) msg "FATAL:  unknown flag '$1'"; usage_run;;
                * ) break;; esac; shift; done