diff --git a/remote-builder-network/README.md b/remote-builder-network/README.md index d04b2d2..f88bc25 100644 --- a/remote-builder-network/README.md +++ b/remote-builder-network/README.md @@ -17,3 +17,12 @@ export EC2_SECRET_KEY="XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" 2. Run `direnv allow` in the root directory + +## Prepare to deploy the network for the first time + +``` +builder-ops create flake.nix +``` + +Before deploying, carefully inspect all the files in this template in order to make sure you understand what it will do. + diff --git a/remote-builder-network/flake.nix b/remote-builder-network/flake.nix index c04e63b..a465f0f 100644 --- a/remote-builder-network/flake.nix +++ b/remote-builder-network/flake.nix @@ -12,6 +12,7 @@ outputs = { self, nixpkgs, nixops, nixops-plugged, utils, ... }: let + networkName = "builder"; eachDefaultEnvironment = f: utils.lib.eachDefaultSystem ( system: @@ -37,7 +38,12 @@ in eachDefaultEnvironment ({ pkgs, system }: { - devShell = import ./shell.nix { inherit pkgs; }; - }); + devShell = import ./shell.nix { inherit pkgs networkName; }; + }) + // { + nixopsConfigurations.default = import ./nixops-configurations {} // { + inherit nixpkgs networkName; + }; + }; } diff --git a/remote-builder-network/nixops-configurations/default.nix b/remote-builder-network/nixops-configurations/default.nix new file mode 100644 index 0000000..5088121 --- /dev/null +++ b/remote-builder-network/nixops-configurations/default.nix @@ -0,0 +1,49 @@ +{ networkName ? "builder" +, networkDescription ? "${networkName} network" +, region ? "us-east-1" +, zone ? "us-east-1b" +, ... +}: + +let + common = { + inherit networkName region zone; + }; + + instances = import ./ec2/instances.nix common; + keys = { + # binary-cache-key = { + # keyCommand = [ "vault" "kv" "get" "-field" "key" "secret/builder/nix-binary-cache" ]; + # }; + }; +in +{ + network.description = networkDescription; + network.enableRollback = true; + + # Currently only legacy state storage is supported + network.storage.legacy = { }; + + # network.storage.s3 = { + # profile = ""; + # region = "us-east-1"; + # key = ""; + # kms_keyid = ""; + # }; + + resources = + import ./ec2/resources.nix common + // import ./iam/resources.nix common + // import ./s3/resources.nix common + // import ./vpc/resources.nix common; + + defaults = instances.defaults; + + builder-1 = { resources, lib, ... }@args: { + deployment = instances.builder args // { inherit keys; }; + imports = [ + ../nixos-configurations/builder + ]; + }; +} + diff --git a/remote-builder-network/nixops-configurations/ec2/instances.nix b/remote-builder-network/nixops-configurations/ec2/instances.nix new file mode 100644 index 0000000..bc215d4 --- /dev/null +++ b/remote-builder-network/nixops-configurations/ec2/instances.nix @@ -0,0 +1,27 @@ +{ networkName +, region +, ... +}: + +let + ec2SpotInstance = { instanceType ? "t2.micro", spotInstancePrice ? 1, resources, lib, ... }: { + targetEnv = "ec2"; + ec2 = { + inherit region instanceType spotInstancePrice; + associatePublicIpAddress = true; + subnetId = resources.vpcSubnets."${networkName}-subnet"; + keyPair = resources.ec2KeyPairs."${networkName}-keypair".name; + securityGroupIds = [ resources.ec2SecurityGroups."${networkName}-sg".name ]; + ebsInitialRootDiskSize = 30; + instanceProfile = resources.iamRoles."${networkName}-role".name; + }; + }; +in +{ + defaults = {}; + + builder = { resources, lib, ... }: + ec2SpotInstance { + inherit resources lib; + }; +} diff --git a/remote-builder-network/nixops-configurations/ec2/resources.nix b/remote-builder-network/nixops-configurations/ec2/resources.nix new file mode 100644 index 0000000..9f63b28 --- /dev/null +++ b/remote-builder-network/nixops-configurations/ec2/resources.nix @@ -0,0 +1,21 @@ +{ networkName +, region +, zone +}: + +{ + ec2KeyPairs."${networkName}-keypair" = { inherit region; }; + + # The --kill-obsolete argument can be used to forcibly delete this resource + # if necessary (nixops' diff engine can't keep track of direct changes) + ec2SecurityGroups."${networkName}-sg" = { resources, lib, ... }: { + inherit region; + vpcId = resources.vpc."${networkName}-vpc"; + rules = builtins.attrValues { + ssh = { toPort = 22; fromPort = 22; sourceIp = "0.0.0.0/0"; }; + http = { toPort = 80; fromPort = 80; sourceIp = "0.0.0.0/0"; }; + https = { toPort = 443; fromPort = 443; sourceIp = "0.0.0.0/0"; }; + }; + }; +} + diff --git a/remote-builder-network/nixops-configurations/iam/resources.nix b/remote-builder-network/nixops-configurations/iam/resources.nix new file mode 100644 index 0000000..0e93dd6 --- /dev/null +++ b/remote-builder-network/nixops-configurations/iam/resources.nix @@ -0,0 +1,44 @@ +{ networkName +, region +, zone +}: + +{ + iamRoles."${networkName}-role" = { + inherit region; + policy = builtins.toJSON { + Statement = [ + # Read from nix binary cache + { + Effect = "Allow"; + Action = [ + "s3:GetObject" + "s3:GetBucketLocation" + ]; + Resource = [ + "arn:aws:s3:::nix-build" + "arn:aws:s3:::nix-build/*" + ]; + } + # Upload to nix binary cache + { + Effect = "Allow"; + Action = [ + "s3:AbortMultipartUpload" + "s3:GetBucketLocation" + "s3:GetObject" + "s3:ListBucket" + "s3:ListBucketMultipartUploads" + "s3:ListMultipartUploadParts" + "s3:PutObject" + ]; + Resource = [ + "arn:aws:s3:::nix-build" + "arn:aws:s3:::nix-build/*" + ]; + } + ]; + }; + }; +} + diff --git a/remote-builder-network/nixops-configurations/s3/resources.nix b/remote-builder-network/nixops-configurations/s3/resources.nix new file mode 100644 index 0000000..580cd54 --- /dev/null +++ b/remote-builder-network/nixops-configurations/s3/resources.nix @@ -0,0 +1,42 @@ +{ networkName +, region ? "us-east-1" +, zone +, ... +}: + +{ + s3Buckets.nix-build = { + inherit region; + name = "nix-build"; + + # Don't delete this bucket ever + persistOnDestroy = true; + + # Not publicly available + # You may want to also set PublicAccessBlock, not currently supported by nixops + website.enabled = false; + + # Save on costs by automatically moving objects to long-term, infrequent access storage after 30 days + lifeCycle = '' + { + "Rules": [ + { + "Status": "Enabled", + "Prefix": "", + "Transitions": [ + { + "Days": 30, + "StorageClass": "GLACIER" + } + ], + "ID": "Glacier", + "AbortIncompleteMultipartUpload": + { + "DaysAfterInitiation": 7 + } + } + ] + } + ''; + }; +} diff --git a/remote-builder-network/nixops-configurations/vpc/resources.nix b/remote-builder-network/nixops-configurations/vpc/resources.nix new file mode 100644 index 0000000..3120658 --- /dev/null +++ b/remote-builder-network/nixops-configurations/vpc/resources.nix @@ -0,0 +1,45 @@ +{ networkName +, region ? "us-east-1" +, zone +, ... +}: + +{ + vpc."${networkName}-vpc" = { + inherit region; + instanceTenancy = "default"; + enableDnsSupport = true; + enableDnsHostnames = true; + cidrBlock = "10.0.0.0/16"; + }; + + vpcSubnets."${networkName}-subnet" = { resources, ... }: { + inherit region zone; + vpcId = resources.vpc."${networkName}-vpc"; + cidrBlock = "10.0.0.0/16"; + mapPublicIpOnLaunch = true; + }; + + vpcRouteTables."${networkName}-route-table" = { resources, ... }: { + inherit region; + vpcId = resources.vpc."${networkName}-vpc"; + }; + + vpcRouteTableAssociations."${networkName}-association" = { resources, ... }: { + inherit region; + subnetId = resources.vpcSubnets."${networkName}-subnet"; + routeTableId = resources.vpcRouteTables."${networkName}-route-table"; + }; + + vpcRoutes."${networkName}-igw-route" = { resources, ... }: { + inherit region; + routeTableId = resources.vpcRouteTables."${networkName}-route-table"; + destinationCidrBlock = "0.0.0.0/0"; + gatewayId = resources.vpcInternetGateways."${networkName}-igw"; + }; + + vpcInternetGateways."${networkName}-igw" = { resources, ... }: { + inherit region; + vpcId = resources.vpc."${networkName}-vpc"; + }; +} diff --git a/remote-builder-network/nixos-configurations/builder/default.nix b/remote-builder-network/nixos-configurations/builder/default.nix new file mode 100644 index 0000000..653c160 --- /dev/null +++ b/remote-builder-network/nixos-configurations/builder/default.nix @@ -0,0 +1,89 @@ +{ pkgs +, lib +, resources +, config +, nodes +, binaryCacheUrl ? s3://nix-build?region=us-east-1 +, binaryCachePrivateKey ? config.deployment.keys.binary-cache-key.path +, binaryCachePublicKey +, ... +}: + +let + # See https://nixos.org/manual/nix/unstable/advanced-topics/post-build-hook.html + uploadToS3Cache = pkgs.writeShellScript "upload-to-s3-cache.sh" '' + set -eu + set -f # disable globbing + export IFS=' ' + + echo "Signing" $OUT_PATHS + echo nix store sign \ + --key-file ${binaryCachePrivateKey} \ + $OUT_PATHS + nix store sign-paths \ + --key-file ${binaryCachePrivateKey} \ + $OUT_PATHS + + echo "Uploading" $OUT_PATHS + echo exec ${pkgs.ts}/bin/ts nix copy \ + --to 's3://nix-build?region=us-east-1¶llel-compression' \ + $OUT_PATHS + exec ${pkgs.ts}/bin/ts nix copy \ + --to 's3://nix-build?region=us-east-1¶llel-compression' \ + $OUT_PATHS + ''; +in +{ + networking.firewall = { + enable = true; + }; + + services = { + openssh = { + enable = true; + # Larger MaxAuthTries helps to avoid ssh 'Too many authentication failures' issue + # See https://github.com/NixOS/nixops/issues/593#issue-203407250 + extraConfig = '' + MaxAuthTries 20 + ''; + openFirewall = true; + }; + }; + + nixpkgs.config.allowUnfree = true; + + boot.loader.grub.device = "/dev/xvda"; + fileSystems."/" = { + label = "nixos"; + fsType = "ext4"; + }; + + environment.systemPackages = + with pkgs; + [ + ]; + + nix.sshServe = { + enable = true; + keys = [ + # Add your own public key here + ]; + protocol = "ssh-ng"; + }; + + # Note that the nix package affects nix.sshServe + nix.package = pkgs.nixFlakes; + nix.extraOptions = '' + post-build-hook = ${uploadToS3Cache} + experimental-features = nix-command flakes + ''; + + nix.binaryCaches = [ + http://cache.nixos.org/ + ]; + nix.binaryCachePublicKeys = [ + "cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=" + binaryCachePublicKey + ]; +} + diff --git a/remote-builder-network/shell.nix b/remote-builder-network/shell.nix index cb4e3d2..eb0f5ba 100644 --- a/remote-builder-network/shell.nix +++ b/remote-builder-network/shell.nix @@ -1,9 +1,10 @@ { pkgs ? import { overlays = import ./overlays.nix; } +, networkName ? "builder" }: let builder = pkgs.callPackage ./. { - networkName = "builder"; + inherit networkName; }; in pkgs.mkShell { @@ -27,7 +28,7 @@ pkgs.mkShell { echo "-------------------------------------" printf "${nc}" echo - ${builder}/bin/builder-help + ${builder}/bin/${networkName}-help # Hook up direnv echo