Merge pull request #194 from input-output-hk/equinix

Equinix
input-output-hk · Jan 19, 2023 · db9a9ca · db9a9ca
2 parents 465fb3b + d99fdae
commit db9a9ca
Show file tree

Hide file tree

Showing 49 changed files with 1,457 additions and 271 deletions.
diff --git a/lib/clusters.nix b/lib/clusters.nix
@@ -22,7 +22,7 @@ lib.listToAttrs (lib.forEach bitteProfiles (bitteProfile: let
   # Currently groupings are viewed as core or awsAsg.
   coreAndPremSimNodes = let
     inherit (cluster);
-    names = map builtins.attrNames [cluster.coreNodes cluster.premNodes cluster.premSimNodes];
+    names = map builtins.attrNames [cluster.coreNodes cluster.awsExtNodes cluster.premNodes cluster.premSimNodes];
     combinedNames = builtins.foldl' (s: v:
       s
       ++ (map (name:
@@ -45,8 +45,9 @@ lib.listToAttrs (lib.forEach bitteProfiles (bitteProfile: let
     .${systemType};
 
   awsCoreNodes = lib.mapAttrs (ourMkSystem "bitteAmazonSystem" coreModules) coreAndPremSimNodes;
+  awsExtNodes = lib.mapAttrs (ourMkSystem "bitteProtoSystem" coreModules) cluster.awsExtNodes;
   premNodes = lib.mapAttrs (ourMkSystem "bitteProtoSystem" coreModules) cluster.premNodes;
-  coreNodes = awsCoreNodes // premNodes;
+  coreNodes = awsCoreNodes // awsExtNodes // premNodes;
 
   awsAutoScalingGroups = lib.mapAttrs (ourMkSystem "bitteAmazonZfsSystem" asgModules) cluster.awsAutoScalingGroups;
 in

diff --git a/lib/default.nix b/lib/default.nix
@@ -30,6 +30,8 @@ in rec {
   ensureDependencies = import ./ensure-dependencies.nix {inherit lib;};
   mkNomadHostVolumesConfig = import ./mk-nomad-host-volumes-config.nix {inherit lib;};
 
+  physicalSpec = import ./physical-spec.nix {inherit lib;};
+
   augmentNomadJob = import ./augment-nomad-job.nix {inherit nixpkgs;};
   mkNomadJobs = ns: envs: let
     pkgs = import nixpkgs {system = "x86_64-linux";};

diff --git a/lib/mk-deploy.nix b/lib/mk-deploy.nix
@@ -12,6 +12,7 @@ assert lib.assertMsg (builtins.typeOf deploySshKey == "string") ''
   deploy' = {
     nodes = builtins.mapAttrs (k: _: let
       cfg = self.nixosConfigurations.${k};
+      deployType = cfg.config.currentCoreNode.deployType or cfg.config.currentAwsAutoScalingGroup.deployType;
     in
       {
         sshUser = "root";
@@ -22,8 +23,10 @@ assert lib.assertMsg (builtins.typeOf deploySshKey == "string") ''
         in
           deploy.lib.${system}.activate.nixos cfg;
       }
-      // (lib.optionalAttrs ((cfg.config.currentCoreNode.deployType or cfg.config.currentAwsAutoScalingGroup.deployType) == "prem") {
+      // (lib.optionalAttrs (builtins.elem deployType ["awsExt" "prem"]) {
         hostname = cfg.config.cluster.name + "-" + cfg.config.networking.hostName;
+      })
+      // (lib.optionalAttrs (deployType == "prem") {
         sshOpts = ["-C" "-o" "StrictHostKeyChecking=no"];
       }))
     self.nixosConfigurations;

diff --git a/lib/mk-system/constants-module.nix b/lib/mk-system/constants-module.nix
@@ -3,7 +3,7 @@
   domain =
     config
     .${
-      if deployType == "aws"
+      if builtins.elem deployType ["aws" "awsExt"]
       then "cluster"
       else "currentCoreNode"
     }

diff --git a/lib/physical-spec.nix b/lib/physical-spec.nix
@@ -0,0 +1,187 @@
+# A library file used to store physical machine spec configuration which
+# will be potentially re-used across a number of clusters.
+{lib, ...}: {
+  _file = ./physical-spec.nix;
+  equinix = let
+    legacyProperties = {
+      properties = {
+        mountpoint = "legacy";
+      };
+    };
+
+    datasets = {
+      "zpool/root" = legacyProperties;
+      "zpool/nix" = legacyProperties;
+      "zpool/home" = legacyProperties;
+      "zpool/var" = legacyProperties;
+      "zpool/cache" = legacyProperties;
+      "zpool/nomad" = legacyProperties;
+      "zpool/containers" = legacyProperties;
+      "zpool/docker" = legacyProperties;
+    };
+
+    mounts = [
+      {
+        dataset = "zpool/root";
+        point = "/";
+      }
+      {
+        dataset = "zpool/nix";
+        point = "/nix";
+      }
+      {
+        dataset = "zpool/var";
+        point = "/var";
+      }
+      {
+        dataset = "zpool/cache";
+        point = "/cache";
+      }
+      {
+        dataset = "zpool/nomad";
+        point = "/var/lib/nomad";
+      }
+      {
+        dataset = "zpool/containers";
+        point = "/var/lib/containers";
+      }
+      {
+        dataset = "zpool/docker";
+        point = "/var/lib/docker";
+      }
+      {
+        dataset = "zpool/home";
+        point = "/home";
+      }
+    ];
+  in {
+    "c3.small.x86" = {
+      cpr_storage = {
+        disks = [
+          {
+            device = "/dev/disk/by-packet-category/boot0";
+            partitions = [
+              {
+                label = "BIOS";
+                number = 1;
+                size = "4096";
+              }
+              {
+                label = "BOOT";
+                number = 2;
+                size = "512M";
+              }
+              {
+                label = "SWAP";
+                number = 3;
+                size = "3993600";
+              }
+              {
+                label = "ROOT";
+                number = 4;
+                size = 0;
+              }
+            ];
+          }
+        ];
+        filesystems = [
+          {
+            mount = {
+              device = "/dev/disk/by-packet-category/boot0-part2";
+              format = "ext4";
+              point = "/boot";
+              create.options = ["-L" "BOOT"];
+            };
+          }
+          {
+            mount = {
+              device = "/dev/disk/by-packet-category/boot0-part3";
+              format = "swap";
+              point = "none";
+              create.options = ["-L" "SWAP"];
+            };
+          }
+        ];
+      };
+
+      cpr_zfs = {
+        inherit datasets mounts;
+        pools = {
+          zpool = {
+            pool_properties = {};
+            vdevs = [
+              {
+                disk = [
+                  "/dev/disk/by-packet-category/boot1"
+                  "/dev/disk/by-packet-category/boot0-part4"
+                ];
+              }
+            ];
+          };
+        };
+      };
+    };
+
+    "m3.small.x86" = {
+      cpr_storage = {
+        disks = [
+          {
+            device = "/dev/disk/by-packet-category/boot0";
+            partitions = [
+              {
+                label = "BIOS";
+                number = 1;
+                size = "512M";
+              }
+              {
+                label = "SWAP";
+                number = 2;
+                size = "3993600";
+              }
+              {
+                label = "ROOT";
+                number = 3;
+                size = 0;
+              }
+            ];
+          }
+        ];
+        filesystems = [
+          {
+            mount = {
+              device = "/dev/disk/by-packet-category/boot0-part1";
+              format = "vfat";
+              point = "/boot";
+              create.options = ["32" "-n" "EFI"];
+            };
+          }
+          {
+            mount = {
+              device = "/dev/disk/by-packet-category/boot0-part2";
+              format = "swap";
+              point = "none";
+              create.options = ["-L" "SWAP"];
+            };
+          }
+        ];
+      };
+
+      cpr_zfs = {
+        inherit datasets mounts;
+        pools = {
+          zpool = {
+            pool_properties = {};
+            vdevs = [
+              {
+                disk = [
+                  "/dev/disk/by-packet-category/boot1"
+                  "/dev/disk/by-packet-category/boot0-part3"
+                ];
+              }
+            ];
+          };
+        };
+      };
+    };
+  };
+}
diff --git a/lib/terralib.nix b/lib/terralib.nix
@@ -32,6 +32,9 @@ in rec {
 
   nullRoute = nullRoute' // {destination_ipv6_cidr_block = null;};
 
+  # Common ssh args used in TF provisioning scripts
+  sshArgs = "-C -oConnectTimeout=5 -oUserKnownHostsFile=/dev/null -oNumberOfPasswordPrompts=0 -oServerAliveInterval=60 -oControlPersist=600 -oStrictHostKeyChecking=no";
+
   aws = {
     # asgVpcs returns a vpc attr struct when provided config.cluster.
     # Example attr struct:

diff --git a/modules/consul.nix b/modules/consul.nix
@@ -610,11 +610,11 @@ in {
         # the machine role (isClient) rather than cfg.server which is not
         # granular enough in this case.
         certChainFile =
-          if (deployType != "aws" && !isClient)
+          if (!(builtins.elem deployType ["aws" "awsExt"]) && !isClient)
           then pkiFiles.serverCertChainFile
           else pkiFiles.certChainFile;
         certKeyFile =
-          if (deployType != "aws" && !isClient)
+          if (!(builtins.elem deployType ["aws" "awsExt"]) && !isClient)
           then pkiFiles.serverKeyFile
           else pkiFiles.keyFile;
         preScript = let

diff --git a/modules/docker-registry.nix b/modules/docker-registry.nix
@@ -16,12 +16,12 @@
   domain =
     config
     .${
-      if deployType == "aws"
+      if builtins.elem deployType ["aws" "awsExt"]
       then "cluster"
       else "currentCoreNode"
     }
     .domain;
-  isSops = deployType == "aws";
+  isSops = builtins.elem deployType ["aws" "awsElem"];
   relEncryptedFolder = last (builtins.split "-" (toString config.secrets.encryptedRoot));
   cfg = config.services.docker-registry;
 in {

diff --git a/modules/monitoring.nix b/modules/monitoring.nix
@@ -35,12 +35,12 @@
   domain =
     config
     .${
-      if deployType == "aws"
+      if builtins.elem deployType ["aws" "awsExt"]
       then "cluster"
       else "currentCoreNode"
     }
     .domain;
-  isSops = deployType == "aws";
+  isSops = builtins.elem deployType ["aws" "awsExt"];
   cfg = config.services.monitoring;
 
   relEncryptedFolder = let

diff --git a/modules/nomad.nix b/modules/nomad.nix
@@ -1234,11 +1234,11 @@ in {
 
       serviceConfig = let
         certChainFile =
-          if (deployType != "aws" && cfg.server.enabled)
+          if (!(builtins.elem deployType ["aws" "awsExt"]) && cfg.server.enabled)
           then pkiFiles.serverCertChainFile
           else pkiFiles.certChainFile;
         certKeyFile =
-          if (deployType != "aws" && cfg.server.enabled)
+          if (!(builtins.elem deployType ["aws" "awsExt"]) && cfg.server.enabled)
           then pkiFiles.serverKeyFile
           else pkiFiles.keyFile;
         start-pre = pkgs.writeBashChecked "nomad-start-pre" (''

diff --git a/modules/promtail.nix b/modules/promtail.nix
@@ -26,12 +26,12 @@
     scrape_configs = [
       {
         ec2_sd_configs =
-          if deployType == "aws"
+          if builtins.elem deployType ["aws" "awsExt"]
           then [{inherit (config.cluster) region;}]
           else [];
 
         job_name =
-          if deployType == "aws"
+          if builtins.elem deployType ["aws" "awsExt"]
           then "ec2-logs"
           else "prem-logs";
 
@@ -74,10 +74,10 @@
             {
               job = "systemd-journal";
             }
-            // lib.optionalAttrs (deployType == "aws") {
+            // lib.optionalAttrs (builtins.elem deployType ["aws" "awsExt"]) {
               inherit (config.cluster) region;
             }
-            // lib.optionalAttrs (deployType != "aws") {
+            // lib.optionalAttrs (!(builtins.elem deployType ["aws" "awsExt"])) {
               inherit datacenter;
             };
           max_age = "12h";