diff --git a/nix/pkgs.nix b/nix/pkgs.nix index c0c4b598d76..b56d08660e7 100644 --- a/nix/pkgs.nix +++ b/nix/pkgs.nix @@ -84,6 +84,11 @@ final: prev: with final; { supervisord-workbench-nix = { workbench ? pkgs.workbench, ... }@args: pkgs.callPackage ./workbench/supervisor.nix args; + docker-workbench-cabal = + { workbench ? pkgs.workbench, ... }@args: pkgs.callPackage ./workbench/docker.nix (args // { useCabalRun = true; }); + docker-workbench-nix = + { workbench ? pkgs.workbench, ... }@args: pkgs.callPackage ./workbench/docker.nix args; + all-profiles-json = (pkgs.callPackage ./workbench/supervisor.nix {}).all-profiles.JSON; # An instance of the workbench, specialised to the supervisord backend and a profile, @@ -103,6 +108,20 @@ final: prev: with final; { inherit batchName profileName supervisord-workbench cardano-node-rev; }; + docker-workbench-for-profile = + { batchName ? customConfig.localCluster.batchName + , profileName ? customConfig.localCluster.profileName + , useCabalRun ? false + , workbenchDevMode ? false + , profiled ? false + , docker-workbench ? pkgs.callPackage ./workbench/docker.nix { inherit useCabalRun; } + , cardano-node-rev ? null + }: + pkgs.callPackage ./workbench/docker-run.nix + { + inherit batchName profileName docker-workbench cardano-node-rev; + }; + # Disable failing python uvloop tests python38 = prev.python38.override { packageOverrides = pythonFinal: pythonPrev: { diff --git a/nix/workbench/app.sh b/nix/workbench/app.sh index f15bb4ddd5e..663e4fa09f6 100644 --- a/nix/workbench/app.sh +++ b/nix/workbench/app.sh @@ -35,7 +35,7 @@ app() { , image: \"$imageName:$imageTag\" , networks: { \"cardano-node-network\": { - ipv4_address: \"172.16.22.1\(.value.i)\" + ipv4_address: \"172.22.\(.value.i / 254 | floor).\(.value.i % 254 + 1)\" } } , ports: [\"\(.value.port):\(.value.port)\"] @@ -44,11 +44,16 @@ app() { , \"LOCAL-\(.value.name):/var/cardano-node/local\" ] , environment: [ - \"HOST_ADDR=172.16.22.1\(.value.i)\" + \"HOST_ADDR=172.22.\(.value.i / 254 | floor).\(.value.i % 254 + 1)\" , \"PORT=\(.value.port)\" , \"DATA_DIR=/var/cardano-node/local\" , \"NODE_CONFIG=/var/cardano-node/local/config.json\" , \"NODE_TOPOLOGY=/var/cardano-node/local/topology.json\" + , \"SOCKET_PATH=/var/cardano-node/local/node.socket\" + , \"RTS_FLAGS=+RTS -N2 -I0 -A16m -qg -qb --disable-delayed-os-memory-return -RTS\" + , \"SHELLEY_KES_KEY=/var/cardano-node/local/../genesis/node-keys/node-kes\(.value.i).skey\" + , \"SHELLEY_VRF_KEY=/var/cardano-node/local/../genesis/node-keys/node-vrf\(.value.i).skey\" + , \"SHELLEY_OPCERT=/var/cardano-node/local/../genesis/node-keys/node\(.value.i).opcert\" ] } } @@ -64,9 +69,9 @@ app() { , ipam: { driver: \"default\" , config: [{ - subnet: \"172.16.22.0/24\" - , ip_range: \"172.16.22.0/24\" - , gateway: \"172.16.22.1\" + subnet: \"172.22.0.0/16\" + , ip_range: \"172.22.0.0/16\" + , gateway: \"172.22.255.254\" , aux_addresses: {} }] } @@ -83,7 +88,7 @@ app() { , driver_opts: { type: \"none\" , o: \"bind\" - , device: \"./run/current/\(.value.name)\" + , device: \"./run/\${WB_RUNDIR_TAG:-current}/\(.value.name)\" } } } @@ -95,7 +100,7 @@ app() { , driver_opts: { type: \"none\" , o: \"bind\" - , device: \"./run/current\" + , device: \"./run/\${WB_RUNDIR_TAG:-current}\" } } } diff --git a/nix/workbench/docker-conf.nix b/nix/workbench/docker-conf.nix new file mode 100644 index 00000000000..aeadaba43e1 --- /dev/null +++ b/nix/workbench/docker-conf.nix @@ -0,0 +1,80 @@ +{ pkgs +, lib +, stateDir +, basePort +, node-services +, generator-service + ## Last-moment overrides: +, extraSupervisorConfig +}: + +with lib; + +let + ## + ## supervisorConf :: SupervisorConf + ## + ## Refer to: http://supervisord.org/configuration.html + ## + supervisorConf = + { + supervisord = { + logfile = "${stateDir}/supervisor/supervisord.log"; + pidfile = "${stateDir}/supervisor/supervisord.pid"; + strip_ansi = true; + }; + supervisorctl = {}; + inet_http_server = { + port = "127.0.0.1:9001"; + }; + "rpcinterface:supervisor" = { + "supervisor.rpcinterface_factory" = "supervisor.rpcinterface:make_main_rpcinterface"; + }; + } + // + listToAttrs + (mapAttrsToList (_: nodeSvcSupervisorProgram) node-services) + // + { + "program:generator" = { + directory = "${stateDir}/generator"; + command = "sh start.sh"; + stdout_logfile = "${stateDir}/generator/stdout"; + stderr_logfile = "${stateDir}/generator/stderr"; + autostart = false; + startretries = 0; + }; + } + // + { + "program:tracer" = { + directory = "${stateDir}/tracer"; + command = "sh start.sh"; + stdout_logfile = "${stateDir}/tracer/stdout"; + stderr_logfile = "${stateDir}/tracer/stderr"; + autostart = false; + startretries = 0; + }; + } + // + extraSupervisorConfig; + + ## + ## nodeSvcSupervisorProgram :: NodeService -> SupervisorConfSection + ## + ## Refer to: http://supervisord.org/configuration.html#program-x-section-settings + ## + nodeSvcSupervisorProgram = { nodeSpec, service, ... }: + nameValuePair "program:${nodeSpec.value.name}" { + directory = "${service.value.stateDir}"; + command = "sh start.sh"; + stdout_logfile = "${service.value.stateDir}/stdout"; + stderr_logfile = "${service.value.stateDir}/stderr"; + startretries = 0; + autostart = false; + autorestart = false; + }; + +in + pkgs.writeText "supervisor.conf" + (generators.toINI {} supervisorConf) diff --git a/nix/workbench/docker-run.nix b/nix/workbench/docker-run.nix new file mode 100644 index 00000000000..a94fdbd3359 --- /dev/null +++ b/nix/workbench/docker-run.nix @@ -0,0 +1,157 @@ +let + batchNameDefault = "plain"; + profileNameDefault = "default-bage"; +in + +{ pkgs +, cardanoNodePackages +, docker-workbench +## +, profileName ? profileNameDefault +, batchName ? batchNameDefault +## +, workbenchDevMode ? false +, cardano-node-rev ? "0000000000000000000000000000000000000000" +}: + +let + inherit (docker-workbench) workbench backend cacheDir stateDir basePort; + + with-docker-profile = + { envArgsOverride ? {} }: + workbench.with-profile + { inherit backend profileName; + envArgs = docker-workbench.env-args-base // envArgsOverride; + }; + + inherit + (with-docker-profile {}) + profileNix profile topology genesis; +in + +let + + inherit (profile.value) era composition monetary; + + path = pkgs.lib.makeBinPath path'; + path' = + [ cardanoNodePackages.bech32 pkgs.jq pkgs.gnused pkgs.coreutils pkgs.bash pkgs.moreutils + ] + ## In dev mode, call the script directly: + ++ pkgs.lib.optionals (!workbenchDevMode) + [ workbench.workbench ]; + + interactive-start = pkgs.writeScriptBin "start-cluster" '' + set -euo pipefail + + export PATH=$PATH:${path} + + wb start \ + --batch-name ${batchName} \ + --profile-name ${profileName} \ + --profile ${profile} \ + --cache-dir ${cacheDir} \ + --base-port ${toString basePort} \ + ''${WB_MODE_CABAL:+--cabal} \ + "$@" + ''; + + interactive-stop = pkgs.writeScriptBin "stop-cluster" '' + set -euo pipefail + + wb finish "$@" + ''; + + interactive-restart = pkgs.writeScriptBin "restart-cluster" '' + set -euo pipefail + + wb run restart "$@" && \ + echo "workbench: alternate command for this action: wb run restart" >&2 + ''; + + nodeBuildProduct = + name: + "report ${name}-log $out ${name}/stdout"; + + profile-run = + { trace ? false }: + let + inherit + (with-docker-profile + { envArgsOverride = { cacheDir = "./cache"; stateDir = "./"; }; }) + profileNix profile topology genesis; + + run = pkgs.runCommand "workbench-run-docker-${profileName}" + { requiredSystemFeatures = [ "benchmark" ]; + nativeBuildInputs = with cardanoNodePackages; with pkgs; [ + bash + bech32 + coreutils + gnused + jq + moreutils + nixWrapped + pstree + python3Packages.docker + workbench.workbench + zstd + ]; + } + '' + mkdir -p $out/{cache,nix-support} + cd $out + export HOME=$out + + export WB_BACKEND=docker + export CARDANO_NODE_SOCKET_PATH=$(wb backend get-node-socket-path ${stateDir} node-0) + + cmd=( + wb + ${pkgs.lib.optionalString trace "--trace"} + start + --profile-name ${profileName} + --profile ${profile} + --topology ${topology} + --genesis-cache-entry ${genesis} + --batch-name smoke-test + --base-port ${toString basePort} + --node-source ${cardanoNodePackages.cardano-node.src.origSrc} + --node-rev ${cardano-node-rev} + --cache-dir ./cache + ) + echo "''${cmd[*]}" > $out/wb-start.sh + + time "''${cmd[@]}" 2>&1 | + tee $out/wb-start.log + + ## Convert structure from $out/run/RUN-ID/* to $out/*: + rm -rf cache + rm -f run/{current,-current} + find $out -type s | xargs rm -f + tag=$(cd run; ls) + (cd run; tar c $tag --zstd) > archive.tar.zst + mv run/$tag/* . + rmdir run/$tag run + + cat > $out/nix-support/hydra-build-products < SupervisorConf + mkSupervisorConf = + profile: + pkgs.callPackage ./docker-conf.nix + { inherit (profile) node-services generator-service; + inherit + pkgs lib stateDir + basePort + extraSupervisorConfig; + }; + }; + }; + + all-profiles = + workbench.all-profiles + { inherit backend; + envArgs = backend.env-args-base; + }; +in +{ + inherit cacheDir stateDir basePort; + inherit workbench; + inherit backend; + inherit all-profiles; +} diff --git a/nix/workbench/docker.sh b/nix/workbench/docker.sh new file mode 100755 index 00000000000..e0c1594420d --- /dev/null +++ b/nix/workbench/docker.sh @@ -0,0 +1,250 @@ +usage_docker() { + usage "docker" "Backend: manages a local cluster using 'dockerd'" </dev/null | grep ':9001 ' | wc -l)" != "0";; + + setenv-defaults ) + local usage="USAGE: wb docker $op PROFILE-DIR" + local profile_dir=${1:?$usage} + ;; + + allocate-run ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + while test $# -gt 0 + do case "$1" in + --* ) msg "FATAL: unknown flag '$1'"; usage_docker;; + * ) break;; esac; shift; done + + cp "$dir/profile/docker-compose.yaml" "$dir" + ;; + + describe-run ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage} + + cat <&2 + while test ! -S $socket + do printf "%3d" $i; sleep 1 + i=$((i+1)) + if test $i -ge $patience + then echo + progress "docker" "$(red FATAL): workbench: docker: patience ran out for $(white $node) after ${patience}s, socket $socket" + backend_docker stop-cluster "$dir" + fatal "$node startup did not succeed: check logs in $(dirname $socket)/stdout & stderr" + fi + echo -ne "\b\b\b" + done >&2 + echo " $node up (${i}s)" >&2 + ;; + + start-nodes ) + local usage="USAGE: wb docker $op RUN-DIR [HONOR_AUTOSTART=]" + local dir=${1:?$usage}; shift + local honor_autostart=${1:-} + + local nodes=($(jq_tolist keys "$dir"/node-specs.json)) + + if test -n "$honor_autostart" + then for node in ${nodes[*]} + do jqtest ".\"$node\".autostart" "$dir"/node-specs.json && + # TODO implement selective start + dockerctl start $node; done; + else docker-compose --file "$dir/docker-compose.yaml" up || + fatal "docker not working"; fi + + for node in ${nodes[*]} + do jqtest ".\"$node\".autostart" "$dir"/node-specs.json && + backend_docker wait-node "$dir" $node; done + + if test ! -v CARDANO_NODE_SOCKET_PATH + then export CARDANO_NODE_SOCKET_PATH=$(backend_docker get-node-socket-path "$dir" 'node-0') + fi + ;; + + start ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + if jqtest ".node.tracer" "$dir"/profile.json + then progress "docker" "faking $(yellow cardano-tracer)" + fi;; + + get-node-socket-path ) + local usage="USAGE: wb docker $op RUN-DIR NODE-NAME" + local dir=${1:?$usage} + local node_name=${2:?$usage} + + echo -n $dir/$node_name/node.socket + ;; + + start-generator ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + while test $# -gt 0 + do case "$1" in + --* ) msg "FATAL: unknown flag '$1'"; usage_docker;; + * ) break;; esac; shift; done + + if ! dockerctl start generator + then progress "docker" "$(red fatal: failed to start) $(white generator)" + echo "$(red generator.json) ------------------------------" >&2 + cat "$dir"/tracer/tracer-config.json + echo "$(red tracer stdout) -----------------------------------" >&2 + cat "$dir"/tracer/stdout + echo "$(red tracer stderr) -----------------------------------" >&2 + cat "$dir"/tracer/stderr + echo "$(white -------------------------------------------------)" >&2 + fatal "could not start $(white dockerd)" + fi + backend_docker save-child-pids "$dir";; + + wait-node-stopped ) + local usage="USAGE: wb docker $op RUN-DIR NODE" + local dir=${1:?$usage}; shift + local node=${1:?$usage}; shift + + progress_ne "docker" "waiting until $node stops: ....." + local i=0 + while dockerctl status $node > /dev/null + do echo -ne "\b\b\b\b\b"; printf "%5d" $i >&2; i=$((i+1)); sleep 1 + done >&2 + echo -e "\b\b\b\b\bdone, after $(with_color white $i) seconds" >&2 + ;; + + wait-pools-stopped ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + local i=0 pools=$(jq .composition.n_pool_hosts $dir/profile.json) start_time=$(date +%s) + msg_ne "docker: waiting until all pool nodes are stopped: 000000" + touch $dir/flag/cluster-termination + + for ((pool_ix=0; pool_ix < $pools; pool_ix++)) + do while dockerctl status node-${pool_ix} > /dev/null && + test -f $dir/flag/cluster-termination + do echo -ne "\b\b\b\b\b\b"; printf "%6d" $((i + 1)); i=$((i+1)); sleep 1; done + echo -ne "\b\b\b\b\b\b"; echo -n "node-${pool_ix} 000000" + done >&2 + echo -ne "\b\b\b\b\b\b" + local elapsed=$(($(date +%s) - start_time)) + if test -f $dir/flag/cluster-termination + then echo " All nodes exited -- after $(yellow $elapsed)s" >&2 + else echo " Termination requested -- after $(yellow $elapsed)s" >&2; fi + ;; + + stop-cluster ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + dockerctl stop all || true + + if test -f "${dir}/docker/dockerd.pid" + then kill $(<${dir}/docker/dockerd.pid) $(<${dir}/docker/child.pids) 2>/dev/null + else pkill dockerd + fi + ;; + + cleanup-cluster ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + msg "docker: resetting cluster state in: $dir" + rm -f $dir/*/std{out,err} $dir/node-*/*.socket $dir/*/logs/* 2>/dev/null || true + rm -fr $dir/node-*/state-cluster/;; + + save-child-pids ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + local svpid=$dir/docker/dockerd.pid + local pstree=$dir/docker/ps.tree + pstree -p "$(cat "$svpid")" > "$pstree" + + local pidsfile="$dir"/docker/child.pids + { grep -e '---\|--=' "$pstree" || true; } | + sed 's/^.*--[=-] \([0-9]*\) .*/\1/; s/^[ ]*[^ ]* \([0-9]+\) .*/\1/ + ' > "$pidsfile" + ;; + + save-pid-maps ) + local usage="USAGE: wb docker $op RUN-DIR" + local dir=${1:?$usage}; shift + + local mapn2p=$dir/docker/node2pid.map; echo '{}' > "$mapn2p" + local mapp2n=$dir/docker/pid2node.map; echo '{}' > "$mapp2n" + local pstree=$dir/docker/ps.tree + + for node in $(jq_tolist keys "$dir"/node-specs.json) + do ## dockerd's service PID is the immediately invoked binary, + ## ..which isn't necessarily 'cardano-node', but could be 'time' or 'cabal' or.. + local service_pid=$(dockerctl pid $node) + if test $service_pid = '0' + then continue + elif test -z "$(ps h --ppid $service_pid)" ## Any children? + then local pid=$service_pid ## <-=^^^ none, in case we're running executables directly. + ## ..otherwise, it's a chain of children, e.g.: time -> cabal -> cardano-node + else local pid=$(grep -e "[=-] $(printf %05d $service_pid) " -A5 "$pstree" | + grep -e '---\|--=' | + head -n1 | + sed 's/^.*--[=-] \([0-9]*\) .*/\1/; + s/^[ ]*[^ ]* \([0-9]*\) .*/\1/') + fi + if test -z "$pid" + then warn "docker" "failed to detect PID of $(white $node)"; fi + jq_fmutate "$mapn2p" '. * { "'$node'": '$pid' }' + jq_fmutate "$mapp2n" '. * { "'$pid'": "'$node'" }' + done + ;; + + * ) usage_docker;; esac +} diff --git a/nix/workbench/shell.nix b/nix/workbench/shell.nix index 20c8de32e2d..4dfa0c28cbd 100644 --- a/nix/workbench/shell.nix +++ b/nix/workbench/shell.nix @@ -18,7 +18,7 @@ with lib; -let cluster = pkgs.supervisord-workbench-for-profile { +let cluster = pkgs.docker-workbench-for-profile { inherit profileName useCabalRun profiled; }; inherit (cluster) profile; @@ -28,7 +28,7 @@ let cluster = pkgs.supervisord-workbench-for-profile { do shift; done ## Flush argv[] echo 'workbench shellHook: workbenchDevMode=${toString workbenchDevMode} useCabalRun=${toString useCabalRun} profiled=${toString profiled} profileName=${profileName}' - export WB_BACKEND=supervisor + export WB_BACKEND=docker export WB_SHELL_PROFILE=${profileName} export WB_SHELL_PROFILE_DIR=${profile} @@ -91,6 +91,7 @@ in project.shellFor { cardano-ping cabalWrapped db-analyser + pkgs.docker-compose ghcid haskellBuildUtils pkgs.graphviz diff --git a/nix/workbench/topology.jq b/nix/workbench/topology.jq index 8ad4fddfe04..3b380225111 100644 --- a/nix/workbench/topology.jq +++ b/nix/workbench/topology.jq @@ -1,10 +1,13 @@ -def loopback_node_topology_from_nixops_topology($topo; $i): +def loopback_node_topology_from_nixops_topology($backend; $topo; $i): $topo.coreNodes[$i].producers as $producers | ($producers | map(ltrimstr("node-") | fromjson)) as $prod_indices | { Producers: ( $prod_indices | map - ({ addr: "127.0.0.1" + ({ addr: (if $backend == "supervisor" + then "127.0.0.1" + else ( "172.22." + (((. / 254) | floor) | tostring) + "." + ((. % 254 + 1) | tostring) ) + end) , port: ($basePort + .) , valency: 1 } diff --git a/nix/workbench/topology.sh b/nix/workbench/topology.sh index 75859e8a79a..a7043216405 100644 --- a/nix/workbench/topology.sh +++ b/nix/workbench/topology.sh @@ -119,6 +119,7 @@ case "${op}" in case "$role" in local-bft | local-pool ) + local backend=${WB_BACKEND:-supervisor} args=(-L$global_basedir --slurpfile topology "$topo_dir"/topology-nixops.json --argjson basePort $basePort @@ -127,7 +128,7 @@ case "${op}" in ) jq 'include "topology"; - loopback_node_topology_from_nixops_topology($topology[0]; $i) + loopback_node_topology_from_nixops_topology("$backend"; $topology[0]; $i) ' "${args[@]}";; local-proxy ) local name=$(jq '.name' <<<$prof --raw-output) diff --git a/nix/workbench/wb b/nix/workbench/wb index 35ab36c7f0b..df551500ed9 100755 --- a/nix/workbench/wb +++ b/nix/workbench/wb @@ -21,6 +21,7 @@ global_basedir=${global_basedir:-$(realpath "$(dirname "$0")")} . "$global_basedir"/app.sh . "$global_basedir"/backend.sh +. "$global_basedir"/docker.sh . "$global_basedir"/supervisor.sh usage_main() { @@ -91,7 +92,7 @@ start() { local batch_name= local profile_name= profile= - local backend=supervisor + local backend=${WB_BACKEND:-supervisor} local node_source=. local node_rev= local cabal_mode= diff --git a/shell.nix b/shell.nix index 7cfab1ca141..11aae847c8f 100644 --- a/shell.nix +++ b/shell.nix @@ -71,7 +71,7 @@ let withMainnet = false; useCabalRun = false; }; - cluster = pkgs.supervisord-workbench-for-profile + cluster = pkgs.docker-workbench-for-profile { inherit (devopsShellParams) profileName useCabalRun; };