serve client certs, nixfmt

clusters/cvn/testnet/default.nix

@@ -281,14 +281,15 @@ in {
       # iPXE is only supported on non-Nitro instances, that means we won't
       # get the latest and greates until they fix that...
       # All currently supported instance families with their smallest type:
+
       # "m4.large" = 1;
       # "t2.large" = 0;
       # "m3.large" = 0;
       # "c4.xlarge" = 0;
       # "d2.xlarge" = 0;
       # "r3.large" = 0;
       # "c3.large" = 0;
-      #
+
       # Use NixOS AMI for now
       "t3a.large" = 1;
     } (instanceType: desiredCapacity:

flake.nix

@@ -51,7 +51,5 @@
         overlays = [ self.overlay.x86_64-linux ];
         system = "x86_64-linux";
       };
-    in {
-      inherit (pkgs) nixosModules nixosConfigurations clusters;
-    });
+    in { inherit (pkgs) nixosModules nixosConfigurations clusters; });
 }

lib/clusters.nix

@@ -3,7 +3,8 @@
 let
   inherit (builtins) attrNames readDir mapAttrs;
   inherit (lib)
-    flip pipe mkForce filterAttrs flatten listToAttrs forEach nameValuePair mapAttrs';
+    flip pipe mkForce filterAttrs flatten listToAttrs forEach nameValuePair
+    mapAttrs';
 
   readDirRec = path:
     pipe path [

modules/certgen.nix

@@ -8,61 +8,74 @@ let
 in {
   # FIXME: this leaves the root certificate on each core machine for signing
   #        themselves...
-  #        Add service in nomad stage that does the cleanup by setting the CA
-  #        in Vault and re-issuing the certificates from there.
+  #        Add service that does the cleanup by setting the CA in Vault and
+  #        re-issuing the certificates from there.
   options = {
     services.certgen.enable = mkEnableOption "Enable certificate distributor";
   };
 
   config = {
-  systemd.services.certgen = mkIf config.services.certgen.enable {
-    wantedBy = [ "multi-user.target" ];
-    before = [ "consul.service" "vault.service" ];
-    requiredBy = [ "consul.service" "vault.service" ];
-
-    serviceConfig = {
-      Type = "oneshot";
-      RemainAfterExit = true;
-    };
+    systemd.services.certgen = mkIf config.services.certgen.enable {
+      wantedBy = [ "multi-user.target" ];
+      before = [ "consul.service" "vault.service" ];
+      requiredBy = [ "consul.service" "vault.service" ];
 
-    path = with pkgs; [ consul jq sops cfssl ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
 
-    script = ''
-      set -exuo pipefail
-      pushd /run/keys
+      path = with pkgs; [ consul jq sops cfssl ];
 
-      enc="certs.enc.json"
+      script = ''
+        set -exuo pipefail
+        pushd /run/keys
 
-      echo "Waiting for $PWD/$enc from deployer..."
+        enc="certs.enc.json"
 
-      set +x
-      until [ -s "$enc" ]; do
-        sleep 1
-      done
-      set -x
+        echo "Waiting for $PWD/$enc from deployer..."
 
-      sops -d certs.enc.json | cfssljson -bare
+        set +x
+        until [ -s "$enc" ]; do
+          sleep 1
+        done
+        set -x
 
-      cat ca.pem core-*.pem > all.pem
-      cp ca.pem /etc/ssl/certs/ca.pem
-      cp all.pem /etc/ssl/certs/all.pem
+        sops -d certs.enc.json | cfssljson -bare
 
-      # Consul
-      mkdir -p /var/lib/consul/certs
-      cp cert-key.pem /var/lib/consul/certs
-      cp cert.pem /var/lib/consul/certs
+        cat ca.pem core-*.pem > all.pem
+        cp ca.pem /etc/ssl/certs/ca.pem
+        cp all.pem /etc/ssl/certs/all.pem
 
-      # Vault
-      mkdir -p /var/lib/vault/certs
-      cp cert-key.pem /var/lib/vault/certs
-      cp cert.pem /var/lib/vault/certs
-      cp core-*.pem /var/lib/vault/certs
+        # Consul
+        mkdir -p /var/lib/consul/certs
+        cp cert-key.pem /var/lib/consul/certs
+        cp cert.pem /var/lib/consul/certs
 
-      # Nomad
-      mkdir -p /var/lib/nomad/certs
-      cp cert-key.pem /var/lib/nomad/certs
-      cp cert.pem /var/lib/nomad/certs
-    '';
-  };
+        # Vault
+        mkdir -p /var/lib/vault/certs
+        cp cert-key.pem /var/lib/vault/certs
+        cp cert.pem /var/lib/vault/certs
+        cp core-*.pem /var/lib/vault/certs
+
+        # Nomad
+        mkdir -p /var/lib/nomad/certs
+        cp cert-key.pem /var/lib/nomad/certs
+        cp cert.pem /var/lib/nomad/certs
+
+
+        enc="client.enc.json"
+        echo "Waiting for $PWD/$enc from deployer..."
+
+        # Clients
+        set +x
+        until [ -s "$enc" ]; do
+          sleep 1
+        done
+        set -x
+
+        cp "$enc/client.enc.json" "/var/lib/nginx"
+      '';
+    };
   };
 }

modules/consul-policies.nix

@@ -150,7 +150,8 @@ in {
       default = { };
     };
 
-    services.consul-policies.enable = mkEnableOption "Create consul policies on this machine";
+    services.consul-policies.enable =
+      mkEnableOption "Create consul policies on this machine";
   };
 
   config = mkIf config.services.consul-policies.enable {

modules/nomad.nix

@@ -179,7 +179,7 @@ in {
           };
 
           chrootEnv = mkOption {
-            type = nullOr ( attrsOf str );
+            type = nullOr (attrsOf str);
             default = null;
             example = { "/usr/bin/env" = "/usr/bin/env"; };
             description = ''
@@ -212,7 +212,7 @@ in {
           };
 
           meta = mkOption {
-            type = nullOr ( attrsOf str );
+            type = nullOr (attrsOf str);
             default = null;
             description = ''
               Specifies a key-value map that annotates with user-defined metadata.

modules/vault-agent.nix

@@ -23,7 +23,7 @@ in {
       serviceConfig = {
         Type = "exec";
         RemainAfterExit = true;
-      ExecStart =  "${pkgs.systemd}/bin/systemctl restart vault-agent.service";
+        ExecStart = "${pkgs.systemd}/bin/systemctl restart vault-agent.service";
       };
     };
 

overlay.nix

@@ -26,8 +26,8 @@ in final: prev: {
     };
   });
 
-  ipxe = prev.callPackage ./pkgs/ipxe.nix { 
-    inherit self; 
+  ipxe = prev.callPackage ./pkgs/ipxe.nix {
+    inherit self;
     embedScript = prev.writeText "ipxe" ''
       #!ipxe
 

pkgs/ec2-ipxe.nix

@@ -1,4 +1,4 @@
-{ipxe, writeText, syslinux}:
+{ ipxe, writeText, syslinux }:
 let
   embedScript = writeText "ipxe" ''
     #!ipxe
@@ -10,13 +10,13 @@ let
     route ||
     chain -ar http://169.254.169.254/latest/user-data
   '';
-in
-  (ipxe.overrideAttrs (old: {
-    makeFlags = [
-      "ECHO_E_BIN_ECHO=echo" "ECHO_E_BIN_ECHO_E=echo" # No /bin/echo here.
-      "ISOLINUX_BIN_LIST=${syslinux}/share/syslinux/isolinux.bin"
-      "LDLINUX_C32=${syslinux}/share/syslinux/ldlinux.c32"
-      # fix for https://github.com/danderson/netboot/pull/117
-      "EMBEDDED_IMAGE=${embedScript}"
-    ];
-  }))
+in (ipxe.overrideAttrs (old: {
+  makeFlags = [
+    "ECHO_E_BIN_ECHO=echo"
+    "ECHO_E_BIN_ECHO_E=echo" # No /bin/echo here.
+    "ISOLINUX_BIN_LIST=${syslinux}/share/syslinux/isolinux.bin"
+    "LDLINUX_C32=${syslinux}/share/syslinux/ldlinux.c32"
+    # fix for https://github.com/danderson/netboot/pull/117
+    "EMBEDDED_IMAGE=${embedScript}"
+  ];
+}))

pkgs/ipxe.nix

@@ -1,7 +1,5 @@
-{ self, stdenv, lib, fetchFromGitHub, perl, cdrkit, syslinux, xz, openssl, gnu-efi, mtools
-, embedScript ? null
-, additionalTargets ? {}
-}:
+{ self, stdenv, lib, fetchFromGitHub, perl, cdrkit, syslinux, xz, openssl
+, gnu-efi, mtools, embedScript ? null, additionalTargets ? { } }:
 
 let
   targets = additionalTargets // lib.optionalAttrs stdenv.isx86_64 {
@@ -15,9 +13,8 @@ let
     "bin/ipxe.lkrn" = null;
     "bin/undionly.kpxe" = null;
   };
-in
 
-stdenv.mkDerivation rec {
+in stdenv.mkDerivation rec {
   pname = "ipxe";
   version = "586b723733904c0825844582dd19a44c71bc972b";
 
@@ -30,12 +27,12 @@ stdenv.mkDerivation rec {
 
   NIX_CFLAGS_COMPILE = "-Wno-error";
 
-  makeFlags =
-    [ "ECHO_E_BIN_ECHO=echo" "ECHO_E_BIN_ECHO_E=echo" # No /bin/echo here.
-      "ISOLINUX_BIN_LIST=${syslinux}/share/syslinux/isolinux.bin"
-      "LDLINUX_C32=${syslinux}/share/syslinux/ldlinux.c32"
-    ] ++ lib.optional (embedScript != null) "EMBEDDED_IMAGE=${embedScript}";
-
+  makeFlags = [
+    "ECHO_E_BIN_ECHO=echo"
+    "ECHO_E_BIN_ECHO_E=echo" # No /bin/echo here.
+    "ISOLINUX_BIN_LIST=${syslinux}/share/syslinux/isolinux.bin"
+    "LDLINUX_C32=${syslinux}/share/syslinux/ldlinux.c32"
+  ] ++ lib.optional (embedScript != null) "EMBEDDED_IMAGE=${embedScript}";
 
   enabledOptions = [
     "PING_CMD"
@@ -61,9 +58,8 @@ stdenv.mkDerivation rec {
   installPhase = ''
     mkdir -p $out
     ${lib.concatStringsSep "\n" (lib.mapAttrsToList (from: to:
-      if to == null
-      then "cp -v ${from} $out"
-      else "cp -v ${from} $out/${to}") targets)}
+      if to == null then "cp -v ${from} $out" else "cp -v ${from} $out/${to}")
+      targets)}
 
     # Some PXE constellations especially with dnsmasq are looking for the file with .0 ending
     # let's provide it as a symlink to be compatible in this case.
@@ -72,11 +68,11 @@ stdenv.mkDerivation rec {
 
   enableParallelBuilding = true;
 
-  meta = with stdenv.lib;
-    { description = "Network boot firmware";
-      homepage = "https://ipxe.org/";
-      license = licenses.gpl2;
-      maintainers = with maintainers; [ ehmry ];
-      platforms = [ "x86_64-linux" "i686-linux" ];
-    };
+  meta = with stdenv.lib; {
+    description = "Network boot firmware";
+    homepage = "https://ipxe.org/";
+    license = licenses.gpl2;
+    maintainers = with maintainers; [ ehmry ];
+    platforms = [ "x86_64-linux" "i686-linux" ];
+  };
 }

profiles/consul/client.nix

@@ -1,7 +1,5 @@
-{...}: {
+{ ... }: {
   imports = [ ./default.nix ];
 
-  services.consul = {
-    enable = true;
-  };
+  services.consul = { enable = true; };
 }

profiles/consul/server.nix

@@ -1,6 +1,6 @@
 { lib, pkgs, config, ... }:
 let
-  inherit (lib) mapAttrsToList ;
+  inherit (lib) mapAttrsToList;
   inherit (config.cluster) instances region;
   instance = instances.${config.networking.hostName};
   inherit (instance) privateIP;

profiles/core.nix

@@ -15,9 +15,5 @@
 
   environment.etc."ipxe".text = "${pkgs.ipxe}";
 
-  environment.systemPackages = with pkgs; [
-    sops
-    awscli
-    cachix
-  ];
+  environment.systemPackages = with pkgs; [ sops awscli cachix ];
 }

profiles/nginx.nix

@@ -9,7 +9,9 @@ in {
 
     security.acme.acceptTerms = lib.mkForce true;
     security.acme.email = lib.mkForce "michael.fellinger@iohk.io";
-    security.acme.certs."ipxe.${config.cluster.domain}".keyType = "rsa4096";
+    # security.acme.certs."ipxe.${config.cluster.domain}".keyType = "rsa4096";
+
+    # TODO: distribute this across core nodes to remove the SPOF
 
     services.nginx = {
       # enableReload = true;
@@ -82,17 +84,20 @@ in {
       };
     };
 
-    systemd.tmpfiles.rules = let flakeBall = pkgs.runCommand "flake.tar.xz" {} ''
-      tar cJf $out -C ${self.outPath}/ .
-    '';
+    systemd.tmpfiles.rules = let
+      flakeBall = pkgs.runCommand "flake.tar.xz" { } ''
+        tar cJf $out -C ${self.outPath}/ .
+      '';
     in [
       "d /var/lib/nginx/nixos-images 0755 nginx nginx -"
       "Z /var/lib/nginx 0755 nginx nginx -"
       "L+ /var/lib/nginx/nixos-images/source - - - - ${self.outPath}"
       "L+ /var/lib/nginx/nixos-images/source.tar.xz - - - - ${flakeBall}"
+      "L+ /var/lib/nginx/nixos-images/client.enc.json - - - - /var/lib/nginx/client.enc.json"
     ];
 
     systemd.services.image-builder = {
+      description = "Make sure we have client machines built for caching";
       after = [ "vault-agent.service" "network-online.target" ];
       requires = [ "vault-agent.service" ];
       wantedBy = [ "nginx.service" "multi-user.target" ];
@@ -111,9 +116,7 @@ in {
           AWS_DEFAULT_REGION VAULT_CACERT VAULT_ADDR VAULT_FORMAT;
       };
 
-      path = with pkgs; [
-        vault-bin glibc gawk gnugrep coreutils nixFlakes
-      ];
+      path = with pkgs; [ vault-bin glibc gawk gnugrep coreutils nixFlakes ];
 
       script = ''
         set -euo pipefail