CAD-2839 supervisord: simpler and cleaner config computation; consoli…

…date paths
IntersectMBO · Apr 8, 2021 · d7fc17f · d7fc17f
1 parent 1b41599
commit d7fc17f
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 30 deletions.
diff --git a/nix/supervisord-cluster/default.nix b/nix/supervisord-cluster/default.nix
@@ -63,8 +63,7 @@ let
     };
 
   supervisorConf = pkgs.callPackage ./supervisor-conf.nix
-    { inherit (topology) nodeSpecs;
-      inherit (node-setups) nodeSetups;
+    { inherit (node-setups) nodeSetups;
       inherit
         pkgs lib stateDir
         basePort
@@ -103,7 +102,7 @@ EOF
       ' ${profileJSONFile} --raw-output
 
     rm -rf     ${stateDir}
-    mkdir -p ./${stateDir}
+    mkdir -p ./${stateDir}/supervisor
 
     PATH=$PATH:${path}
     ${defCardanoExesBash}
@@ -117,7 +116,7 @@ EOF
         (name: nodeSetup:
           ''
           jq . ${__toFile "${name}-node-config.json"
-            (__toJSON nodeSetup.nodeConfig)} > \
+            (__toJSON nodeSetup.nodeService.nodeConfig)} > \
              ${stateDir}/${name}/config.json
 
           jq . ${__toFile "${name}-service-config.json"
@@ -135,9 +134,9 @@ EOF
           ''
         ))}
 
+    cp ${supervisorConf} ${stateDir}/supervisor/supervisord.conf
     ${pkgs.python3Packages.supervisor}/bin/supervisord \
-        --config ${__trace "supervisorConfig: ${supervisorConf} "
-                   supervisorConf} $@
+        --config ${supervisorConf} $@
 
     ## Wait for socket activation:
     #
@@ -146,10 +145,10 @@ EOF
     while [ ! -S $CARDANO_NODE_SOCKET_PATH ]; do echo "Waiting 5 seconds for bft node to start"; sleep 5; done
 
     echo 'Recording node pids..'
-    ${pkgs.psmisc}/bin/pstree -Ap $(cat ${stateDir}/supervisord.pid) |
+    ${pkgs.psmisc}/bin/pstree -Ap $(cat ${stateDir}/supervisor/supervisord.pid) |
     grep 'cabal.*cardano-node' |
     sed -e 's/^.*-+-cardano-node(\([0-9]*\))-.*$/\1/' \
-      > ${stateDir}/cardano-node.pids
+      > ${stateDir}/supervisor/cardano-node.pids
 
     ${optionalString (!profile.genesis.single_shot)
      ''
@@ -166,11 +165,11 @@ EOF
   stop = pkgs.writeScriptBin "stop-cluster" ''
     set -euo pipefail
     ${pkgs.python3Packages.supervisor}/bin/supervisorctl stop all
-    if [ -f ${stateDir}/supervisord.pid ]
+    if [ -f ${stateDir}/supervisor/supervisord.pid ]
     then
-      kill $(<${stateDir}/supervisord.pid) $(<${stateDir}/cardano-node.pids)
+      kill $(<${stateDir}/supervisor/supervisord.pid) $(<${stateDir}/supervisor/cardano-node.pids)
       echo "Cluster terminated!"
-      rm -f ${stateDir}/supervisord.pid ${stateDir}/cardano-node.pids
+      rm -f ${stateDir}/supervisor/supervisord.pid ${stateDir}/supervisor/cardano-node.pids
     else
       echo "Cluster is not running!"
     fi

diff --git a/nix/supervisord-cluster/node-setups.nix b/nix/supervisord-cluster/node-setups.nix
@@ -11,17 +11,27 @@
 with lib;
 
 let
+
+  ## The AWS node is started with:
+  ## cardano-node run
+  ## --config /nix/store/nywkyj205skkqy27ip3p0678977kxq0b-config-1-0.json
+  ## --database-path /var/lib/cardano-node/db-bench-dense-k51-10ep-2000kU-500kD-6600esec
+  ## --topology /nix/store/sb8gn8wb4s8m7a4pmkb6hvlr4fhy5vn2-topology.yaml
+  ## --shelley-vrf-key /var/lib/keys/cardano-node-vrf-signing
+  ## --shelley-kes-key /var/lib/keys/cardano-node-kes-signing
+  ## --shelley-operational-certificate /var/lib/keys/cardano-node-operational-cert
+  ## +RTS -l-agu -t --machine-readable -RTS +RTS -N2 -A16m -qg -qb -M14336.000000M -RTS
   ##
   ## nodeSpecServiceConfig :: NodeSpec -> ServiceConfig
   ##
   nodeSpecServiceConfig =
     { name, i, kind, port, isProducer }:
     {
       stateDir       = stateDir + "/${name}";
-      socketPath     = "${stateDir}/${name}/node.socket";
-      topology       = "${stateDir}/${name}/topology.json";
-      nodeConfigFile = "${stateDir}/${name}/config.json";
-      dbPrefix       = "db-${name}";
+      socketPath     = "node.socket";
+      topology       = "topology.json";
+      nodeConfigFile = "config.json";
+      dbPrefix       = "db";
       port           = nodeIndexToNodePort i;
       nodeConfig =
         lib.recursiveUpdate defaultLogConfig
@@ -74,9 +84,9 @@ let
               };
           }).${profile.era});
     } // optionalAttrs isProducer {
-      operationalCertificate = "${stateDir}/shelley/node-keys/node${toString i}.opcert";
-      kesKey                 = "${stateDir}/shelley/node-keys/node-kes${toString i}.skey";
-      vrfKey                 = "${stateDir}/shelley/node-keys/node-vrf${toString i}.skey";
+      operationalCertificate = "../shelley/node-keys/node${toString i}.opcert";
+      kesKey                 = "../shelley/node-keys/node-kes${toString i}.skey";
+      vrfKey                 = "../shelley/node-keys/node-vrf${toString i}.skey";
     } // optionalAttrs useCabalRun {
       executable = "cabal run exe:cardano-node --";
     };
@@ -121,8 +131,7 @@ let
         nodeServiceConfig = nodeSpecServiceConfig nodeSpec;
         nodeService = nodeServiceConfigService nodeServiceConfig;
       in {
-        inherit nodeSpec nodeServiceConfig;
-        inherit (nodeService) nodeConfig;
+        inherit nodeSpec nodeServiceConfig nodeService;
 
         startupScript =
           pkgs.writeScript "cardano-node"

diff --git a/nix/supervisord-cluster/supervisor-conf.nix b/nix/supervisord-cluster/supervisor-conf.nix
@@ -2,7 +2,6 @@
 , lib
 , stateDir
 , basePort
-, nodeSpecs                ## :: Map NodeName NodeSpec
 , nodeSetups               ## :: Map NodeName NodeSetup
   ## Last-moment overrides:
 , extraSupervisorConfig
@@ -12,23 +11,28 @@ with lib;
 
 let
   ##
-  ## nodeSpecSupervisorProgram :: NodeSpec -> SupervisorConfSection
+  ## nodeSetupSupervisorProgram :: NodeSpec -> SupervisorConfSection
   ##
-  nodeSpecSupervisorProgram = { name, i, kind, port, isProducer }:
-    nameValuePair "program:${kind}-${toString i}" {
-      command        = "${nodeSetups."${name}".startupScript}";
-      stdout_logfile = "${stateDir}/${name}/stdout";
-      stderr_logfile = "${stateDir}/${name}/stderr";
+  ## Refer to: http://supervisord.org/configuration.html#program-x-section-settings
+  ##
+  nodeSetupSupervisorProgram = { nodeSpec, nodeService, startupScript, ... }:
+    nameValuePair "program:${nodeSpec.name}" {
+      directory      = "${nodeService.stateDir}";
+      command        = "${startupScript}";
+      stdout_logfile = "${nodeService.stateDir}/stdout";
+      stderr_logfile = "${nodeService.stateDir}/stderr";
     };
 
   ##
   ## supervisorConf :: SupervisorConf
   ##
+  ## Refer to: http://supervisord.org/configuration.html
+  ##
   supervisorConf =
     {
       supervisord = {
-        logfile = "${stateDir}/supervisord.log";
-        pidfile = "${stateDir}/supervisord.pid";
+        logfile = "${stateDir}/supervisor/supervisord.log";
+        pidfile = "${stateDir}/supervisor/supervisord.pid";
         strip_ansi = true;
       };
       supervisorctl = {};
@@ -41,7 +45,7 @@ let
     }
     //
     listToAttrs
-      (mapAttrsToList (_: nodeSpecSupervisorProgram) nodeSpecs)
+      (mapAttrsToList (_: nodeSetupSupervisorProgram) nodeSetups)
     //
     {
       "program:webserver" = {