wip: prem deployType support

input-output-hk · Jan 26, 2022 · f52cdbb · f52cdbb
1 parent 9542cc7
commit f52cdbb
Show file tree

Hide file tree

Showing 9 changed files with 83 additions and 48 deletions.
diff --git a/lib/clusters.nix b/lib/clusters.nix
@@ -5,37 +5,40 @@
 
 lib.listToAttrs (lib.forEach clusterFiles (file:
   let
-    inherit (_proto.config) tf;
-
     _proto = (mkSystem {
       inherit pkgs self inputs;
       modules = [ file hydrateModule ];
     }).bitteProtoSystem;
 
+    inherit (_proto.config) tf cluster;
+
     # Separating core and premSim nodes may cause bitte-cli tooling to break.
     # Currently groupings are viewed as core or awsAsg.
-    # May be able to split premSim nodes out going forward.
-    coreAndPremSimNodes = assert (lib.assertMsg (!builtins.any
-      (e: builtins.elem e (builtins.attrNames _proto.config.cluster.coreNodes))
-      (builtins.attrNames _proto.config.cluster.premSimNodes)) ''
-        ERROR
-        trace: ERROR  -->  premSimNodes may not have the same names as coreNodes
-      '');
-      _proto.config.cluster.premSimNodes // _proto.config.cluster.coreNodes;
-
-    coreNodes = lib.mapAttrs (nodeName: coreNode:
-      (mkSystem {
-        inherit pkgs self inputs nodeName;
-        modules = [ { networking.hostName = lib.mkForce nodeName; } file hydrateModule ]
-          ++ coreNode.modules;
-      }).bitteAmazonSystem) coreAndPremSimNodes;
-
-    awsAutoScalingGroups = lib.mapAttrs (nodeName: awsAutoScalingGroup:
-      (mkSystem {
-        inherit pkgs self inputs nodeName;
-        modules = [ file ] ++ awsAutoScalingGroup.modules;
-      }).bitteAmazonZfsSystem) _proto.config.cluster.awsAutoScalingGroups;
-
-  in lib.nameValuePair _proto.config.cluster.name {
+    coreAndPremSimNodes =
+       let
+         inherit (cluster);
+         names = map builtins.attrNames [ cluster.coreNodes cluster.premNodes cluster.premSimNodes ];
+         combinedNames = builtins.foldl' (s: v:
+           s ++ (map (name:
+             if (builtins.elem name s) then
+               throw "Duplicate node name: ${name}"
+             else
+               name) v)) [ ] names;
+       in builtins.seq combinedNames (cluster.coreNodes // cluster.premSimNodes);
+
+    ourMkSystem = attr: nodeName: coreNode: (mkSystem {
+      inherit pkgs self inputs ;
+      nodeName = nodeName;
+      modules = [ { networking.hostName = lib.mkForce nodeName; } file hydrateModule ]
+        ++ coreNode.modules;
+    }).${attr};
+
+    awsCoreNodes = lib.mapAttrs (ourMkSystem "bitteAmazonSystem") coreAndPremSimNodes;
+    premNodes = lib.mapAttrs (ourMkSystem "bitteProtoSystem") cluster.premNodes;
+    coreNodes = awsCoreNodes // premNodes;
+
+    awsAutoScalingGroups = lib.mapAttrs (ourMkSystem "bitteAmazonZfsSystem") cluster.awsAutoScalingGroups;
+
+  in lib.nameValuePair cluster.name {
     inherit _proto tf coreNodes awsAutoScalingGroups;
   }))
diff --git a/lib/mk-deploy.nix b/lib/mk-deploy.nix
@@ -10,14 +10,19 @@ assert lib.assertMsg (builtins.typeOf deploySshKey == "string") ''
 let
 
   deploy' = {
-    sshUser = "root";
-    sshOpts = [ "-C" "-i" "${deploySshKey}" "-o" "StrictHostKeyChecking=no" ];
-    nodes = builtins.mapAttrs (k: _: {
+    nodes = builtins.mapAttrs (k: _: let
+      cfg = self.nixosConfigurations.${k};
+    in {
+      sshUser = "root";
+      sshOpts = [ "-C" "-o" "StrictHostKeyChecking=no" "-i" "${deploySshKey}"];
       profiles.system.user = "root";
       profiles.system.path =
-        let inherit (self.nixosConfigurations.${k}.pkgs) system;
-        in deploy.lib.${system}.activate.nixos self.nixosConfigurations.${k};
-    }) self.nixosConfigurations;
+        let inherit (cfg.pkgs) system;
+        in deploy.lib.${system}.activate.nixos cfg;
+    } // (lib.optionalAttrs (cfg.config.currentCoreNode.deployType  == "prem") {
+      hostname = cfg.config.cluster.name + "-" + cfg.config.networking.hostName;
+      sshOpts = [ "-C" "-o" "StrictHostKeyChecking=no" ];
+    })) self.nixosConfigurations;
   };
 
 in {

diff --git a/lib/mk-system.nix b/lib/mk-system.nix
@@ -66,7 +66,6 @@ let
       ../profiles/ami-base-config.nix
     ];
   });
-
 in {
   inherit bitteSystem bitteProtoSystem bitteAmazonSystem
     bitteAmazonSystemBaseAMI bitteAmazonZfsSystem bitteAmazonZfsSystemBaseAMI;

diff --git a/modules/promtail.nix b/modules/promtail.nix
@@ -13,7 +13,7 @@ let
 
     clients = [{
       url =
-        "http://${config.cluster.coreNodes.monitoring.privateIP}:3100/loki/api/v1/push";
+        "http://${config.cluster.nodes.monitoring.privateIP}:3100/loki/api/v1/push";
     }];
 
     positions = { filename = "/var/lib/promtail/positions.yaml"; };

diff --git a/modules/terraform.nix b/modules/terraform.nix
@@ -203,6 +203,12 @@ let
 
         terraformOrganization = lib.mkOption { type = with lib.types; str; };
 
+        nodes = lib.mkOption {
+          type = with lib.types; attrsOf coreNodeType;
+          internal = true;
+          default = cfg.coreNodes // cfg.premSimNodes // cfg.premNodes;
+        };
+
         coreNodes = lib.mkOption {
           type = with lib.types; attrsOf coreNodeType;
           default = { };
@@ -213,6 +219,11 @@ let
           default = { };
         };
 
+        premNodes = lib.mkOption {
+          type = with lib.types; attrsOf coreNodeType;
+          default = { };
+        };
+
         awsAutoScalingGroups = lib.mkOption {
           type = with lib.types; attrsOf awsAutoScalingGroupType;
           default = { };
@@ -907,7 +918,20 @@ in {
     currentCoreNode = lib.mkOption {
       internal = true;
       type = with lib.types; nullOr attrs;
-      default = cfg.coreNodes."${nodeName}" or cfg.premSimNodes."${nodeName}" or null;
+      default = let
+        names =
+          map builtins.attrNames [ cfg.coreNodes cfg.premNodes cfg.premSimNodes ];
+        combinedNames = builtins.foldl' (s: v:
+          s ++ (map (name:
+            if (builtins.elem name s) then
+              throw "Duplicate node name: ${name}"
+            else
+              name) v)) [ ] names;
+      in builtins.seq combinedNames
+      (cfg.coreNodes."${nodeName}" or
+       cfg.premNodes."${nodeName}" or
+       cfg.premSimNodes."${nodeName}" or
+      null);
     };
 
     currentAwsAutoScalingGroup = lib.mkOption {

diff --git a/modules/vault-agent.nix b/modules/vault-agent.nix
@@ -138,12 +138,13 @@ in {
       wantedBy = [ "multi-user.target" ];
 
       environment = {
-        inherit (config.environment.variables) AWS_DEFAULT_REGION;
         CONSUL_HTTP_ADDR = "127.0.0.1:8500";
         VAULT_ADDR = cfg.vaultAddress;
         VAULT_SKIP_VERIFY = "true";
         VAULT_FORMAT = "json";
-      };
+      } // (lib.optionalAttrs (config.environment.variables ? "AWS_DEFAULT_REGION") {
+        inherit (config.environment.variables) AWS_DEFAULT_REGION;
+      });
 
       path = with pkgs; [ vault-bin ];
 

diff --git a/profiles/auxiliaries/nix.nix b/profiles/auxiliaries/nix.nix
@@ -1,4 +1,4 @@
-{ pkgs, config, self, ... }: {
+{ pkgs, self, ... }: {
   nix = {
     package = pkgs.nixFlakes;
     gc.automatic = true;
@@ -20,11 +20,10 @@
     };
     systemFeatures = [ "recursive-nix" "nixos-test" ];
 
-    binaryCaches = [ "https://hydra.iohk.io" config.cluster.s3Cache ];
+    binaryCaches = [ "https://hydra.iohk.io" ];
 
     binaryCachePublicKeys = [
       "hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ="
-      config.cluster.s3CachePubKey
     ];
   };
 }
diff --git a/profiles/auxiliaries/telegraf.nix b/profiles/auxiliaries/telegraf.nix
@@ -168,7 +168,7 @@ in {
       outputs = {
         influxdb = {
           database = "telegraf";
-          urls = [ "http://${config.cluster.coreNodes.monitoring.privateIP}:8428" ];
+          urls = [ "http://${config.cluster.nodes.monitoring.privateIP}:8428" ];
         };
       };
     };

diff --git a/profiles/bootstrap/default.nix b/profiles/bootstrap/default.nix
@@ -193,9 +193,10 @@ in {
       };
 
       environment = {
-        inherit (config.environment.variables)
-          AWS_DEFAULT_REGION VAULT_CACERT VAULT_FORMAT VAULT_ADDR;
-      };
+        inherit (config.environment.variables) VAULT_CACERT VAULT_FORMAT VAULT_ADDR;
+      } // (lib.optionalAttrs (config.environment.variables ? "AWS_DEFAULT_REGION") {
+        inherit (config.environment.variables) AWS_DEFAULT_REGION;
+      });
 
       path = with pkgs; [ sops rage vault-bin consul nomad coreutils jq curl ];
 
@@ -284,10 +285,12 @@ in {
       };
 
       environment = {
-        inherit (config.environment.variables) AWS_DEFAULT_REGION NOMAD_ADDR;
+        inherit (config.environment.variables) NOMAD_ADDR;
         CURL_CA_BUNDLE = if deployType == "aws" then pkiFiles.certChainFile
                          else pkiFiles.serverCertChainFile;
-      };
+      } // (lib.optionalAttrs (config.environment.variables ? "AWS_DEFAULT_REGION") {
+        inherit (config.environment.variables) AWS_DEFAULT_REGION;
+      });
 
       path = with pkgs; [ curl sops rage coreutils jq nomad vault-bin gawk ];
 
@@ -361,9 +364,10 @@ in {
       };
 
       environment = {
-        inherit (config.environment.variables)
-          AWS_DEFAULT_REGION VAULT_CACERT VAULT_FORMAT VAULT_ADDR;
-      };
+        inherit (config.environment.variables) VAULT_CACERT VAULT_FORMAT VAULT_ADDR;
+      } // (lib.optionalAttrs (config.environment.variables ? "AWS_DEFAULT_REGION") {
+        inherit (config.environment.variables) AWS_DEFAULT_REGION;
+      });
 
       path = with pkgs; [
         consul