WIP WIP

nix/supervisord-cluster/default.nix

@@ -8,19 +8,23 @@ in
 , workbench
 , lib
 , bech32
-, customConfig
-
-, useCabalRun
+, basePort              ? basePortDefault
+, stateDir              ? stateDirDefault
+, cacheDir              ? cacheDirDefault
+, extraSupervisorConfig ? {}
+, useCabalRun           ? false
+, workbenchDevMode      ? false
+, enableEKG             ? true
+##
+, profileName           ? profileNameDefault
+, batchName             ? "plain"
 , ...
 }:
-with lib; with customConfig.localCluster;
+with lib;
 let
   backend =
     rec
-    { ## First, generic items:
-      inherit basePort;
-      staggerPorts = true;
-
+    { ## Generic Nix bits:
       topologyForNode =
         { profile, nodeSpec }:
         let inherit (nodeSpec) name i; in
@@ -58,7 +62,7 @@ let
             ];
           });
 
-      ## Backend-specific:
+      ## Backend-specific Nix bits:
       supervisord =
         {
           inherit
@@ -77,54 +81,6 @@ let
                 extraSupervisorConfig;
             };
         };
-
-      ## Actions:
-      deploy =
-        profile:
-        ''
-        cat <<EOF
-workbench:  starting cluster:
-  - state dir:       ${stateDir}
-  - profile JSON:    ${profile.JSON}
-  - node specs:      ${profile.node-specs.JSON}
-  - topology:        ${profile.topology.files}/topology-nixops.json ${profile.topology.files}/topology.pdf
-  - node port base:  ${toString basePort}
-  - EKG URLs:        http://localhost:${toString (basePort + supervisord.portShiftEkg)}/
-  - Prometheus URLs: http://localhost:${toString (basePort + supervisord.portShiftPrometheus)}/metrics
-
-EOF
-
-        ${__concatStringsSep "\n"
-          (flip mapAttrsToList profile.node-services
-            (name: svc:
-              ''
-              mkdir -p ${stateDir}/${name}
-              cp ${svc.nodeSpec.JSON}      ${stateDir}/${name}/spec.json
-              cp ${svc.serviceConfig.JSON} ${stateDir}/${name}/service-config.json
-              cp ${svc.nodeConfig.JSON}    ${stateDir}/${name}/config.json
-              cp ${svc.topology.JSON}      ${stateDir}/${name}/topology.json
-              cp ${svc.startupScript}      ${stateDir}/${name}/start.sh
-              ''
-            ))}
-
-        SUPERVISOR_CONF=${supervisord.mkSupervisorConf profile}
-        cp $SUPERVISOR_CONF ${stateDir}/supervisor/supervisord.conf
-        ${pkgs.python3Packages.supervisor}/bin/supervisord \
-            --config $SUPERVISOR_CONF $@
-        '';
-
-      activate =
-        profile:
-        ''
-        if test ! -v CARDANO_NODE_SOCKET_PATH
-        then export  CARDANO_NODE_SOCKET_PATH=$(wb local get-node-socket-path ${stateDir})
-        fi
-
-        wb local wait-for-node-socket
-        wb local record-node-pids     "${stateDir}"
-
-        echo 'workbench:  cluster started. Run `stop-cluster` to stop'
-        '';
     };
 
   environment =
@@ -156,38 +112,64 @@ let
   start = pkgs.writeScriptBin "start-cluster" ''
     set -euo pipefail
 
-    PATH=$PATH:${path}
+    export PATH=$PATH:${path}
+    export WORKBENCH_BACKEND=${../workbench}/supervisor.sh
+
+    wb backend assert-is supervisor
+
+    wb backend assert-stopped
 
-    batch_name=plain
+    batch_name=${batchName}
 
     while test $# -gt 0
     do case "$1" in
         --batch-name ) batch_name=$2; shift;;
         --trace | --debug ) set -x;;
-        --trace-wb | --trace-workbench ) export workbench_extra_flags=--trace;;
+        --trace-wb | --trace-workbench ) export WORKBENCH_EXTRA_FLAGS=--trace;;
         * ) break;; esac; shift; done
 
-    wb local assert-stopped
-
-    workbench-prebuild-executables || return 1
-
-    wb profile describe ${profileName}
+    workbench-prebuild-executables
 
     if test -e "${stateDir}" -a ! -L "${stateDir}"
     then echo "workbench ERROR:  state directory exists, but is not a symlink -- please remove it or choose another:  ${stateDir}"; exit 1; fi
 
     wb_run_allocate_args=(
-        --cachedir "${cacheDir}"
+        --cachedir      "${cacheDir}"
+        --base-port      ${toString basePort}
+        --stagger-ports
+        --port-shift-ekg        100
+        --port-shift-prometheus 200
       )
-    wb run "''${wb_run_allocate_args[@]}" allocate $batch_name ${profile.name}
+    wb run allocate $batch_name ${profile.name} "''${wb_run_allocate_args[@]}"
 
     ln -sf "$(wb run current-path)" "${stateDir}"
     mkdir -p                        "${stateDir}"/supervisor
 
+    ${__concatStringsSep "\n"
+      (flip mapAttrsToList profile.node-services
+        (name: svc:
+          ''
+          cp ${svc.serviceConfig.JSON} ${stateDir}/${name}/service-config.json
+          cp ${svc.nodeConfig.JSON}    ${stateDir}/${name}/config.json
+          cp ${svc.topology.JSON}      ${stateDir}/${name}/topology.json
+          cp ${svc.startupScript}      ${stateDir}/${name}/start.sh
+          ''
+        ))}
+
+    SUPERVISOR_CONF=${supervisord.mkSupervisorConf profile}
+    cp $SUPERVISOR_CONF ${stateDir}/supervisor/supervisord.conf
+    ${pkgs.python3Packages.supervisor}/bin/supervisord \
+        --config $SUPERVISOR_CONF $@
+    if test ! -v CARDANO_NODE_SOCKET_PATH
+    then export  CARDANO_NODE_SOCKET_PATH=$(wb local get-node-socket-path ${stateDir})
+    fi
+
     wb run start $(wb run current-name)
 
-    ${backend.deploy   profile}
-    ${backend.activate profile}
+    wb backend wait-for-local-node-socket
+    wb supervisor record-node-pids     "${stateDir}"
+
+    echo 'workbench:  cluster started. Run `stop-cluster` to stop'
   '';
   stop = pkgs.writeScriptBin "stop-cluster" ''
     set -euo pipefail

nix/workbench/backend.sh

@@ -0,0 +1,50 @@
+usage_backend() {
+     usage "backend" "Abstract over cluster backend operations" <<EOF
+    is-running       Test if cluster is running
+
+    get-node-socket-path RUNDIR
+                     Given a run directory, print the node socket path
+                       for 'cardano-cli'
+
+    wait-for-local-node-socket
+                     Wait until CARDANO_NODE_SOCKET_PATH becomes a valid socket
+
+    record-extended-env-config ENV-JSON [ENV-CONFIG-OPTS..]
+                     Extend the environment JSON file with backend-specific
+                       environment config
+
+    assert-is BACKEND-NAME
+                     Check that the current backend is as expected
+
+    assert-stopped   Assert that cluster is not running
+EOF
+}
+
+backend() {
+backend op=${1:-$(usage_backend)}
+
+case "${op}" in
+    is-running )                 $WORKBENCH_BACKEND "$@";;
+    get-node-socket-path )       $WORKBENCH_BACKEND "$@";;
+    wait-for-local-node-socket ) $WORKBENCH_BACKEND "$@";;
+    record-extended-env-config ) $WORKBENCH_BACKEND "$@";;
+    describe-run )               $WORKBENCH_BACKEND "$@";;
+
+    assert-is )
+        local usage="USAGE: wb run $op BACKEND-NAME"
+        local name=${1:?$usage}
+
+        ## Check the backend echoes own name:
+        local actual_name=$($WORKBENCH_BACKEND name)
+        if test "$actual_name" != "$name"
+        then fatal "Workbench is broken:  '$WORKBENCH_BACKEND name' returned:  '$actual_name'"; fi
+        ;;
+
+    assert-stopped )
+        $backend is-running &&
+          fatal "backend reports that cluster is already running. Please stop it first!" ||
+          true
+        ;;
+
+    * ) usage_backend;; esac
+}

nix/workbench/default.nix

@@ -81,14 +81,14 @@ let
     ''
     echo 'workbench:  dev mode enabled, calling wb directly from checkout (instead of using Nix store)' >&2
 
-    workbench_cardano_node_repo_root=$(git rev-parse --show-toplevel)
-    workbench_extra_flags=
+    WORKBENCH_CARDANO_NODE_REPO_ROOT=$(git rev-parse --show-toplevel)
+    WORKBENCH_EXTRA_FLAGS=
 
     function wb() {
-      $workbench_cardano_node_repo_root/nix/workbench/wb --set-mode ${checkoutWbMode} $workbench_extra_flags "$@"
+      $WORKBENCH_CARDANO_NODE_REPO_ROOT/nix/workbench/wb --set-mode ${checkoutWbMode} $WORKBENCH_EXTRA_FLAGS "$@"
     }
 
-    export workbench_cardano_node_repo_root workbench_extra_flags
+    export WORKBENCH_CARDANO_NODE_REPO_ROOT WORKBENCH_EXTRA_FLAGS
     export -f wb
 
     ''}

nix/workbench/lib.sh

@@ -4,12 +4,21 @@ to_jsonlist() {
     done | jq --slurp '.'
 }
 
+jq_tolist() {
+    local exp=$1; shift
+    jq "$exp | join (\" \")" --raw-output "$@"
+}
+
 jq_fmutate() {
     local f=$1; shift
     test -f "$f" || { echo '{}' > "$f"; }
     jq "$@" "$f" | sponge "$f"
 }
 
+jq_check_json() {
+    jq '.' "$1" >/dev/null
+}
+
 __usage() {
     local op=$1 desc=$2
     cat >&2 <<EOF

nix/workbench/profile.sh

@@ -5,10 +5,8 @@ usage_profile() {
     compose NAME..        Create a profile composed from named profiles
     get NAME              Get contents of named profile
     describe NAME         Print a human description of a profile
-    node-specs PROFILE-JSON PORT-BASE STAGGER-PORTS
-                          Print node specs JSON for the given profile;
-                            If STAGGER-PORTS is true, the assigned ports will
-                            be incrementally distinc for each node spec
+    node-specs PROFILE-JSON ENV-JSON
+                          Print node specs JSON for the given profile and environment
 EOF
 }
 
@@ -84,15 +82,13 @@ case "${op}" in
           ' --raw-output);;
 
     node-specs )
-        local usage="USAGE: wb profile node-specs PROFILE-JSON PORT-BASE STAGGER-PORTS"
+        local usage="USAGE: wb profile node-specs PROFILE-JSON ENV-JSON"
         local profile_json=${1:?$usage}
-        local port_base=${2:?$usage}
-        local stagger_ports=${3:?$usage}
+        local env_json=${2:?$usage}
 
         args=(
             "$profile_json"
-            --argjson port_base     $port_base
-            --argjson stagger_ports $stagger_ports
+            --slurpfile env "$env_json"
         )
         jq '. as $prof
            | $prof.composition.n_bft_hosts  as $n_bfts
@@ -121,9 +117,9 @@ case "${op}" in
                     { name:       "node-\(.["i"])"
                     , isProducer: ([.kind == "bft", .kind == "pool"] | any)
                     , port:
-                      (if $stagger_ports
-                       then $port_base + .i
-                       else $port_base
+                      (if $env[0].stagger_ports
+                       then $env[0].base_port + .i
+                       else $env[0].base_port
                        end)
                     }))
            | map({ key: .name, value: .})

nix/workbench/profiles/node-services.nix

@@ -165,7 +165,6 @@ let
         };
 
         topology = rec {
-          ## XXX: hard-code for local case!
           JSON  = backend.topologyForNode { inherit profile nodeSpec; };
           value = __fromJSON (__readFile JSON);
         };