Add network definitions for AWS resources and the builder instance

remote-builder-network/README.md

@@ -17,3 +17,12 @@ export EC2_SECRET_KEY="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
 
 2. Run `direnv allow` in the root directory
 
+
+## Prepare to deploy the network for the first time
+
+```
+builder-ops create flake.nix
+```
+
+Before deploying, carefully inspect all the files in this template in order to make sure you understand what it will do.
+

remote-builder-network/flake.nix

@@ -12,6 +12,7 @@
 
   outputs = { self, nixpkgs, nixops, nixops-plugged, utils, ... }:
     let
+      networkName = "builder";
       eachDefaultEnvironment = f: utils.lib.eachDefaultSystem
         (
           system:
@@ -37,7 +38,12 @@
     in
     eachDefaultEnvironment
       ({ pkgs, system }: {
-        devShell = import ./shell.nix { inherit pkgs; };
-      });
+        devShell = import ./shell.nix { inherit pkgs networkName; };
+      })
+    // {
+      nixopsConfigurations.default = import ./nixops-configurations {} // {
+        inherit nixpkgs networkName;
+      };
+    };
 }
 

remote-builder-network/nixops-configurations/default.nix

@@ -0,0 +1,49 @@
+{ networkName ? "builder"
+, networkDescription ? "${networkName} network"
+, region ? "us-east-1"
+, zone ? "us-east-1b"
+, ...
+}:
+
+let
+  common = {
+    inherit networkName region zone;
+  };
+
+  instances = import ./ec2/instances.nix common;
+  keys = {
+    # binary-cache-key = {
+    #   keyCommand = [ "vault" "kv" "get" "-field" "key" "secret/builder/nix-binary-cache" ];
+    # };
+  };
+in
+{
+  network.description = networkDescription;
+  network.enableRollback = true;
+
+  # Currently only legacy state storage is supported
+  network.storage.legacy = { };
+
+  # network.storage.s3 = {
+  #   profile = "";
+  #   region = "us-east-1";
+  #   key = "";
+  #   kms_keyid = "";
+  # };
+
+  resources =
+    import ./ec2/resources.nix common
+    // import ./iam/resources.nix common
+    // import ./s3/resources.nix common
+    // import ./vpc/resources.nix common;
+
+  defaults = instances.defaults;
+
+  builder-1 = { resources, lib, ... }@args: {
+    deployment = instances.builder args // { inherit keys; };
+    imports = [
+      ../nixos-configurations/builder
+    ];
+  };
+}
+

remote-builder-network/nixops-configurations/ec2/instances.nix

@@ -0,0 +1,27 @@
+{ networkName
+, region
+, ...
+}:
+
+let
+  ec2SpotInstance = { instanceType ? "t2.micro", spotInstancePrice ? 1, resources, lib, ... }: {
+    targetEnv = "ec2";
+    ec2 = {
+      inherit region instanceType spotInstancePrice;
+      associatePublicIpAddress = true;
+      subnetId = resources.vpcSubnets."${networkName}-subnet";
+      keyPair = resources.ec2KeyPairs."${networkName}-keypair".name;
+      securityGroupIds = [ resources.ec2SecurityGroups."${networkName}-sg".name ];
+      ebsInitialRootDiskSize = 30;
+      instanceProfile = resources.iamRoles."${networkName}-role".name;
+    };
+  };
+in
+{
+  defaults = {};
+
+  builder = { resources, lib, ... }:
+    ec2SpotInstance {
+      inherit resources lib;
+    };
+}

remote-builder-network/nixops-configurations/ec2/resources.nix

@@ -0,0 +1,21 @@
+{ networkName
+, region
+, zone
+}:
+
+{
+  ec2KeyPairs."${networkName}-keypair" = { inherit region; };
+
+  # The --kill-obsolete argument can be used to forcibly delete this resource
+  # if necessary (nixops' diff engine can't keep track of direct changes)
+  ec2SecurityGroups."${networkName}-sg" = { resources, lib, ... }: {
+    inherit region;
+    vpcId = resources.vpc."${networkName}-vpc";
+    rules = builtins.attrValues {
+      ssh = { toPort = 22; fromPort = 22; sourceIp = "0.0.0.0/0"; };
+      http = { toPort = 80; fromPort = 80; sourceIp = "0.0.0.0/0"; };
+      https = { toPort = 443; fromPort = 443; sourceIp = "0.0.0.0/0"; };
+    };
+  };
+}
+

remote-builder-network/nixops-configurations/iam/resources.nix

@@ -0,0 +1,44 @@
+{ networkName
+, region
+, zone
+}:
+
+{
+  iamRoles."${networkName}-role" = {
+    inherit region;
+    policy = builtins.toJSON {
+      Statement = [
+        # Read from nix binary cache
+        {
+          Effect = "Allow";
+          Action = [
+            "s3:GetObject"
+            "s3:GetBucketLocation"
+          ];
+          Resource = [
+            "arn:aws:s3:::nix-build"
+            "arn:aws:s3:::nix-build/*"
+          ];
+        }
+        # Upload to nix binary cache
+        {
+          Effect = "Allow";
+          Action = [
+            "s3:AbortMultipartUpload"
+            "s3:GetBucketLocation"
+            "s3:GetObject"
+            "s3:ListBucket"
+            "s3:ListBucketMultipartUploads"
+            "s3:ListMultipartUploadParts"
+            "s3:PutObject"
+          ];
+          Resource = [
+            "arn:aws:s3:::nix-build"
+            "arn:aws:s3:::nix-build/*"
+          ];
+        }
+      ];
+    };
+  };
+}
+

remote-builder-network/nixops-configurations/s3/resources.nix

@@ -0,0 +1,42 @@
+{ networkName
+, region ? "us-east-1"
+, zone
+, ...
+}:
+
+{
+  s3Buckets.nix-build = {
+    inherit region;
+    name = "nix-build";
+
+    # Don't delete this bucket ever
+    persistOnDestroy = true;
+
+    # Not publicly available
+    # You may want to also set PublicAccessBlock, not currently supported by nixops
+    website.enabled = false;
+
+    # Save on costs by automatically moving objects to long-term, infrequent access storage after 30 days
+    lifeCycle = ''
+      {
+        "Rules": [
+           {
+             "Status": "Enabled",
+             "Prefix": "",
+             "Transitions": [
+               {
+                 "Days": 30,
+                 "StorageClass": "GLACIER"
+               }
+             ],
+             "ID": "Glacier",
+             "AbortIncompleteMultipartUpload":
+               {
+                 "DaysAfterInitiation": 7
+               }
+           }
+        ]
+      }
+    '';
+  };
+}

remote-builder-network/nixops-configurations/vpc/resources.nix

@@ -0,0 +1,45 @@
+{ networkName
+, region ? "us-east-1"
+, zone
+, ...
+}:
+
+{
+  vpc."${networkName}-vpc" = {
+    inherit region;
+    instanceTenancy = "default";
+    enableDnsSupport = true;
+    enableDnsHostnames = true;
+    cidrBlock = "10.0.0.0/16";
+  };
+
+  vpcSubnets."${networkName}-subnet" = { resources, ... }: {
+    inherit region zone;
+    vpcId = resources.vpc."${networkName}-vpc";
+    cidrBlock = "10.0.0.0/16";
+    mapPublicIpOnLaunch = true;
+  };
+
+  vpcRouteTables."${networkName}-route-table" = { resources, ... }: {
+    inherit region;
+    vpcId = resources.vpc."${networkName}-vpc";
+  };
+
+  vpcRouteTableAssociations."${networkName}-association" = { resources, ... }: {
+    inherit region;
+    subnetId = resources.vpcSubnets."${networkName}-subnet";
+    routeTableId = resources.vpcRouteTables."${networkName}-route-table";
+  };
+
+  vpcRoutes."${networkName}-igw-route" = { resources, ... }: {
+    inherit region;
+    routeTableId = resources.vpcRouteTables."${networkName}-route-table";
+    destinationCidrBlock = "0.0.0.0/0";
+    gatewayId = resources.vpcInternetGateways."${networkName}-igw";
+  };
+
+  vpcInternetGateways."${networkName}-igw" = { resources, ... }: {
+    inherit region;
+    vpcId = resources.vpc."${networkName}-vpc";
+  };
+}

remote-builder-network/nixos-configurations/builder/default.nix

@@ -0,0 +1,89 @@
+{ pkgs
+, lib
+, resources
+, config
+, nodes
+, binaryCacheUrl ? s3://nix-build?region=us-east-1
+, binaryCachePrivateKey ? config.deployment.keys.binary-cache-key.path
+, binaryCachePublicKey
+, ...
+}:
+
+let
+  # See https://nixos.org/manual/nix/unstable/advanced-topics/post-build-hook.html
+  uploadToS3Cache = pkgs.writeShellScript "upload-to-s3-cache.sh" ''
+    set -eu
+    set -f # disable globbing
+    export IFS=' '
+
+    echo "Signing" $OUT_PATHS
+    echo nix store sign \
+      --key-file ${binaryCachePrivateKey} \
+      $OUT_PATHS
+    nix store sign-paths \
+      --key-file ${binaryCachePrivateKey} \
+      $OUT_PATHS
+
+    echo "Uploading" $OUT_PATHS
+    echo exec ${pkgs.ts}/bin/ts nix copy \
+      --to 's3://nix-build?region=us-east-1&parallel-compression' \
+      $OUT_PATHS
+    exec ${pkgs.ts}/bin/ts nix copy \
+      --to 's3://nix-build?region=us-east-1&parallel-compression' \
+      $OUT_PATHS
+  '';
+in
+{
+  networking.firewall = {
+    enable = true;
+  };
+
+  services = {
+    openssh = {
+      enable = true;
+      # Larger MaxAuthTries helps to avoid ssh 'Too many authentication failures' issue
+      # See https://github.com/NixOS/nixops/issues/593#issue-203407250
+      extraConfig = ''
+        MaxAuthTries 20
+      '';
+      openFirewall = true;
+    };
+  };
+
+  nixpkgs.config.allowUnfree = true;
+
+  boot.loader.grub.device = "/dev/xvda";
+  fileSystems."/" = {
+    label = "nixos";
+    fsType = "ext4";
+  };
+
+  environment.systemPackages =
+    with pkgs;
+    [
+    ];
+
+  nix.sshServe = {
+    enable = true;
+    keys = [
+      # Add your own public key here
+    ];
+    protocol = "ssh-ng";
+  };
+
+  # Note that the nix package affects nix.sshServe
+  nix.package = pkgs.nixFlakes;
+  nix.extraOptions = ''
+    post-build-hook = ${uploadToS3Cache}
+    experimental-features = nix-command flakes
+  '';
+
+  nix.binaryCaches = [
+    http://cache.nixos.org/
+  ];
+  nix.binaryCachePublicKeys = [
+    "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY="
+    binaryCachePublicKey
+  ];
+}
+

remote-builder-network/shell.nix

@@ -1,9 +1,10 @@
 { pkgs ? import <nixpkgs> { overlays = import ./overlays.nix; }
+, networkName ? "builder"
 }:
 
 let
   builder = pkgs.callPackage ./. {
-    networkName = "builder";
+    inherit networkName;
   };
 in
 pkgs.mkShell {
@@ -27,7 +28,7 @@ pkgs.mkShell {
         echo "-------------------------------------"
         printf "${nc}"
         echo
-        ${builder}/bin/builder-help
+        ${builder}/bin/${networkName}-help
 
         # Hook up direnv
         echo