workbench: WIP

IntersectMBO · Feb 7, 2023 · e1f9c3b · e1f9c3b
1 parent 242ff6d
commit e1f9c3b
Show file tree

Hide file tree

Showing 19 changed files with 330 additions and 247 deletions.
diff --git a/Makefile b/Makefile
@@ -47,6 +47,9 @@ ci-report:
 	@echo -e "\033[34mGoals under test\033[0m:  \033[33m$(CI_TARGETS)\033[0m"
 ci-targets:  $(CI_TARGETS)
 
+workbench-internals-walkthrough:
+	emn nix/workbench/doc.org
+
 ##
 ## Base targets:
 ##

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -44,6 +44,10 @@
       url = "github:input-output-hk/plutus-apps";
       flake = false;
     };
+    em = {
+      url = "github:deepfire/em";
+      flake = false;
+    };
 
     # Custom user config (default: empty), eg.:
     # { outputs = {...}: {
@@ -100,6 +104,7 @@
     , std
     , nix2container
     , cardano-automation
+    , em
     , ...
     }@input:
     let
@@ -126,6 +131,7 @@
           inherit customConfig nix2container;
           inherit (tullia.packages.${final.system}) tullia nix-systems;
           bench-data-publish = cardano-automation.outputs.packages.${final.system}."bench-data-publish:exe:bench-data-publish";
+          em = import em { nixpkgsSrcs = nixpkgs.outPath; nixpkgsRev = nixpkgs.rev; };
           gitrev = final.customConfig.gitrev or self.rev or "0000000000000000000000000000000000000000";
           commonLib = lib
             // iohkNix.lib

diff --git a/nix/nixos/cardano-node-service.nix b/nix/nixos/cardano-node-service.nix
@@ -230,6 +230,15 @@ in {
         '';
       };
 
+      isProducer = mkOption {
+        type = types.bool;
+        default = false;
+        description = ''
+          Whether this node is intended to be a producer.
+          Internal option for inter-module communication.
+        '';
+      };
+
       # Byron signing/delegation
 
       signingKey = mkOption {

diff --git a/nix/pkgs.nix b/nix/pkgs.nix
@@ -12,8 +12,8 @@ let
   # A conveniently-parametrisable workbench preset.
   # See https://input-output-hk.github.io/haskell.nix/user-guide/development/
   # The general idea is:
-  # backendName -> useCabalRun -> backend
-  # stateDir -> batchName -> profileName -> backend -> workbench -> runner
+  # 1. backendName -> useCabalRun -> backend
+  # 2. stateDir -> batchName -> profileName -> backend -> workbench -> runner
   # * `workbench` is in case a pinned version of the workbench is needed.
   workbench-runner =
     let backendRegistry =
@@ -29,6 +29,7 @@ let
     , profileName        ? if profileNix != null then profileNix.profileName
                            else customConfig.localCluster.profileName
     , backendName        ? customConfig.localCluster.backendName
+    , basePort           ? customConfig.localCluster.basePort
     , useCabalRun        ? customConfig.localCluster.useCabalRun
     , workbenchDevMode   ? customConfig.localCluster.workbenchDevMode
     , profiled           ? customConfig.profiled
@@ -40,13 +41,13 @@ let
         # override its value. The runner uses the value of `useCabalRun` from
         # the backend to prevent a runner using a different value.
         backend = import (backendRegistry."${backendName}")
-                   { inherit pkgs lib useCabalRun; };
+                   { inherit pkgs lib stateDir basePort useCabalRun; };
     in import ./workbench/backend/runner.nix
       {
         inherit pkgs lib cardanoNodePackages;
         inherit stateDir batchName profileName backend;
         inherit cardano-node-rev;
-        inherit useCabalRun workbench workbenchDevMode;
+        inherit workbench workbenchDevMode;
       };
 
   # Workbench instantiated by parameters from customConfig:

diff --git a/nix/workbench/backend/nixops.nix b/nix/workbench/backend/nixops.nix
@@ -1,5 +1,6 @@
 { pkgs
 , lib
+, basePort
 , ...
 }:
 with lib; with pkgs.commonLib;
@@ -286,10 +287,25 @@ let
         };
       };
     };
+
+  service-modules = {
+    node =
+      { name, config, ... }:
+      { _file = ./nixops.nix;
+        config.services.cardano-node = {
+          signingKey             = mkForce "/var/lib/keys/cardano-node-signing";
+          delegationCertificate  = mkForce "/var/lib/keys/cardano-node-delegation-cert";
+          kesKey                 = mkForce "/var/lib/keys/cardano-node-kes-signing";
+          vrfKey                 = mkForce "/var/lib/keys/cardano-node-vrf-signing";
+          operationalCertificate = mkForce "/var/lib/keys/cardano-node-operational-cert";
+        };
+      };
+  };
 in
 {
-  inherit extraShellPkgs materialise-profile overlay;
-
   name = "nixops";
+
+  inherit extraShellPkgs materialise-profile overlay service-modules basePort;
+
   useCabalRun = false;
 }
diff --git a/nix/workbench/backend/nixops.sh b/nix/workbench/backend/nixops.sh
@@ -59,11 +59,12 @@ case "$op" in
         )
         nixops $crmod "${crmod_args[@]}"
 
-        progress "run | nixops" "cleaning up cluster node state"
-        nixops ssh-for-each       \
-               --deployment $depl \
-               --parallel         \
-               -- rm -rf '"/var/lib/cardano-node/*"'
+        if test $crmod = "modify"
+        then progress "run | nixops" "cleaning up cluster node state"
+             nixops ssh-for-each       \
+                    --deployment $depl \
+                    --parallel         \
+                    -- rm -rf '"/var/lib/cardano-node/*"'; fi
 
         progress "run | nixops" "deploying"
         deploy_args+=(

diff --git a/nix/workbench/backend/nomad.nix b/nix/workbench/backend/nomad.nix
@@ -1,5 +1,7 @@
 { pkgs
 , lib
+
+, basePort
 ## `useCabalRun` not used here like in `supervisor.nix`.
 , ...
 }:
@@ -92,7 +94,7 @@ let
               # Actually always "false", may evolve to a "cloud" flag!
               oneTracerPerNode = false;
             };
-      in pkgs.runCommand "workbench-backend-output-${profileNix.profileName}-${name}"
+      in pkgs.runCommand "workbench-backend-output-${profileNix.profileName}-nomad"
         ({
           ociImagesJSON = ociImages.JSON;
           inherit nomadJobJSON;
@@ -109,5 +111,7 @@ let
     };
 in
 {
-  inherit name extraShellPkgs materialise-profile overlay useCabalRun;
+  name = "nomad";
+
+  inherit extraShellPkgs materialise-profile overlay basePort useCabalRun;
 }
diff --git a/nix/workbench/backend/runner.nix b/nix/workbench/backend/runner.nix
@@ -13,15 +13,14 @@
 ##
 , basePort              ? 30000
 , cacheDir              ? "${__getEnv "HOME"}/.cache/cardano-workbench"
-, useCabalRun
 }:
 let
   backendName = backend.name;
 
   inherit (backend) useCabalRun;
 
   profileNix = workbench.materialise-profile
-    { inherit basePort stateDir useCabalRun profileName; };
+    { inherit basePort stateDir useCabalRun profileName backend; };
   backendNix = backend.materialise-profile
     { inherit stateDir profileNix; };
 in

diff --git a/nix/workbench/backend/supervisor-conf.nix b/nix/workbench/backend/supervisor-conf.nix
@@ -27,22 +27,6 @@ let
       };
     }
     //
-    lib.attrsets.optionalAttrs (unixHttpServerPort != null) {
-      unix_http_server = {
-        file = unixHttpServerPort;
-        chmod = "0777";
-      };
-    }
-    //
-    lib.attrsets.optionalAttrs (inetHttpServerPort != null) {
-      inet_http_server = {
-        port = inetHttpServerPort;
-      };
-    }
-    //
-    listToAttrs
-      (mapAttrsToList (_: nodeSvcSupervisorProgram) (profileNix.node-services))
-    //
     {
       "program:generator" = {
         directory      = "${stateDir}/generator";
@@ -58,7 +42,8 @@ let
       };
     }
     //
-    lib.attrsets.optionalAttrs (profileNix.value.node.tracer) {
+    lib.attrsets.optionalAttrs (profileNix.value.node.tracer)
+    {
       "program:tracer" = {
         directory      = "${stateDir}/tracer";
         command        = "sh start.sh";
@@ -71,25 +56,39 @@ let
         startretries   = 1;
         startsecs      = 1;
       };
-    };
-
-  ##
-  ## nodeSvcSupervisorProgram :: NodeService -> SupervisorConfSection
-  ##
-  ## Refer to: http://supervisord.org/configuration.html#program-x-section-settings
-  ##
-  nodeSvcSupervisorProgram = { nodeSpec, service, ... }:
-    nameValuePair "program:${nodeSpec.value.name}" {
-      directory      = "${service.value.stateDir 0}";
-      command        = "sh start.sh";
-      stdout_logfile = "${service.value.stateDir 0}/stdout";
-      stderr_logfile = "${service.value.stateDir 0}/stderr";
-      stopasgroup    = false;
-      killasgroup    = false;
-      autostart      = false;
-      autorestart    = false;
-      startretries   = 1;
-      startsecs      = 1;
+    }
+    //
+    listToAttrs
+      (flip mapAttrsToList profileNix.node-services
+        (_: { nodeSpec, service, ... }:
+          nameValuePair "program:${nodeSpec.value.name}" {
+            ##
+            ## Refer to: http://supervisord.org/configuration.html#program-x-section-settings
+            ##
+            directory      = "${service.value.stateDir 0}";
+            command        = "sh start.sh";
+            stdout_logfile = "${service.value.stateDir 0}/stdout";
+            stderr_logfile = "${service.value.stateDir 0}/stderr";
+            stopasgroup    = false;
+            killasgroup    = false;
+            autostart      = false;
+            autorestart    = false;
+            startretries   = 1;
+            startsecs      = 1;
+          })
+        )
+    //
+    lib.attrsets.optionalAttrs (unixHttpServerPort != null) {
+      unix_http_server = {
+        file = unixHttpServerPort;
+        chmod = "0777";
+      };
+    }
+    //
+    lib.attrsets.optionalAttrs (inetHttpServerPort != null) {
+      inet_http_server = {
+        port = inetHttpServerPort;
+      };
     };
 
 in {

diff --git a/nix/workbench/backend/supervisor.nix b/nix/workbench/backend/supervisor.nix
@@ -1,11 +1,12 @@
 { pkgs
 , lib
+, stateDir
 , useCabalRun
+, basePort
 , ...
 }:
+with lib;
 let
-  name = "supervisor";
-
   extraShellPkgs = with pkgs;
     [
       python3Packages.supervisor
@@ -33,7 +34,7 @@ let
           inherit pkgs lib stateDir;
           inetHttpServerPort = "127.0.0.1:9001";
         };
-      in pkgs.runCommand "workbench-backend-output-${profileNix.profileName}-${name}"
+      in pkgs.runCommand "workbench-backend-output-${profileNix.profileName}-supervisor"
         {supervisorConfPath = supervisorConf.INI;}
         ''
         mkdir $out
@@ -44,7 +45,30 @@ let
     proTopo: self: super:
     {
     };
+
+  service-modules = {
+    node =
+      { config, ... }:
+      let selfCfg = config.services.cardano-node;
+          i       = toString selfCfg.nodeId;
+      in
+      { #_file = ./supervisor.nix;
+
+        services.cardano-node.stateDir = stateDir + "/node-${i}";
+        services.cardano-node.operationalCertificate =
+          mkIf (selfCfg.isProducer)
+            "../genesis/node-keys/node${i}.opcert";
+        services.cardano-node.kesKey =
+          mkIf (selfCfg.isProducer)
+            "../genesis/node-keys/node-kes${i}.skey";
+        services.cardano-node.vrfKey =
+          mkIf (selfCfg.isProducer)
+            "../genesis/node-keys/node-vrf${i}.skey";
+      };
+  };
 in
 {
-  inherit name extraShellPkgs materialise-profile overlay useCabalRun;
+  name = "supervisor";
+
+  inherit extraShellPkgs materialise-profile overlay basePort useCabalRun service-modules;
 }
diff --git a/nix/workbench/default.nix b/nix/workbench/default.nix
@@ -87,7 +87,6 @@ in {
     topologyFiles
     nodeSpecsJson
     genesisFiles
-    servicesConfig
     services
     profile
     profileData