Convert terraform config to use variables

.github/workflows/terraform-plan.yml

@@ -16,20 +16,20 @@ jobs:
     runs-on: ubuntu-20.04
     environment: ${{ github.base_ref == 'main' && 'Development' || 'Production' }}
     env:
-      DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
-      GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
-      AWS_VPC_ID: ${{ secrets.AWS_VPC_ID }}
-      AWS_SUBNET_PRIMARY: ${{ secrets.AWS_SUBNET_PRIMARY }}
-      AWS_ZONE_PRIMARY: ${{ secrets.AWS_ZONE_PRIMARY }}
-      AWS_SUBNET_SECONDARY0: ${{ secrets.AWS_SUBNET_SECONDARY0 }}
-      AWS_ZONE_SECONDARY0: ${{ secrets.AWS_ZONE_SECONDARY0 }}
-      AWS_SUBNET_SECONDARY1: ${{ secrets.AWS_SUBNET_SECONDARY1 }}
-      AWS_ZONE_SECONDARY1: ${{ secrets.AWS_ZONE_SECONDARY1 }}
-      AWS_SSH_KEY: ${{ secrets.AWS_SSH_KEY }}
-      GIT_REPOSITORY_URL: https://github.com/neu-dsg/dailp-encoding
-      OAUTH_TOKEN: ${{ secrets.OAUTH_TOKEN }}
+      TF_VAR_database_password: ${{ secrets.DATABASE_PASSWORD }}
+      TF_VAR_aws_vpc_id: ${{ secrets.AWS_VPC_ID }}
+      TF_VAR_aws_subnet_primary: ${{ secrets.AWS_SUBNET_PRIMARY }}
+      TF_VAR_aws_zone_primary: ${{ secrets.AWS_ZONE_PRIMARY }}
+      TF_VAR_aws_subnet_secondary0: ${{ secrets.AWS_SUBNET_SECONDARY0 }}
+      TF_VAR_aws_zone_secondary0: ${{ secrets.AWS_ZONE_SECONDARY0 }}
+      TF_VAR_aws_subnet_secondary1: ${{ secrets.AWS_SUBNET_SECONDARY1 }}
+      TF_VAR_aws_zone_secondary1: ${{ secrets.AWS_ZONE_SECONDARY1 }}
+      TF_VAR_aws_ssh_key: ${{ secrets.AWS_SSH_KEY }}
+      TF_VAR_git_repository_url: https://github.com/neu-dsg/dailp-encoding
+      TF_VAR_github_oauth_token: ${{ secrets.OAUTH_TOKEN }}
+      TF_VAR_deployment_stage: ${{ github.base_ref == 'main' && 'dev' || 'prod' }}
+      TF_VAR_cluster_join_token: ${{ secrets.HERCULES_CLUSTER_JOIN_TOKEN }}
       RUST_LOG: info
-      TF_STAGE: ${{ github.base_ref == 'main' && 'dev' || 'prod' }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v2

flake.nix

@@ -19,7 +19,8 @@
   };
 
   outputs = inputs:
-    inputs.utils.lib.eachDefaultSystem (system:
+
+    (inputs.utils.lib.eachDefaultSystem (system:
       let
         pkgs = import inputs.pkgs {
           inherit system;
@@ -211,5 +212,47 @@
               libiconv
             ];
           };
-      });
+      })) // {
+        nixosConfigurations."dailp-ci-runner" = inputs.pkgs.lib.nixosSystem {
+          system = "x86_64-linux";
+          modules = [
+            ({ pkgs, ... }: {
+              # TODO: Move this to a separate module file.
+              networking.hostName = "dailp-ci-runner";
+              ec2.hvm = true;
+              services.hercules-ci-agent.enable = true;
+              services.hercules-ci-agent.concurrentTasks = 4;
+              systemd.services.install-secrets = {
+                enable = true;
+                before = [ "hercules-ci-agent.service" ];
+                wantedBy = [ "hercules-ci-agent.service" ];
+                script = ''
+                  install --directory \
+                          --owner hercules-ci-agent \
+                          --group nogroup \
+                          --mode 0700 \
+                          /var/lib/hercules-ci-agent/secrets \
+                          ;
+                  install --mode 0400 \
+                          --owner hercules-ci-agent \
+                          /var/keys/cluster_join_token \
+                          /var/lib/hercules-ci-agent/secrets/cluster-join-token.key \
+                          ;
+                  install --mode 0400 \
+                          --owner hercules-ci-agent \
+                          /var/keys/binary_caches_json \
+                          /var/lib/hercules-ci-agent/secrets/binary-caches.json \
+                          ;
+                '';
+                serviceConfig.Type = "oneshot";
+              };
+
+              # Limit journal size
+              services.journald.extraConfig = ''
+                SystemMaxUse=1024M
+              '';
+            })
+          ];
+        };
+      };
 }

terraform/ci-configuration.nix

@@ -0,0 +1,67 @@
+{ hostName, primaryAddress, secondary0Address ? null, secondary1Address ? null
+, ... }:
+let
+  nixpkgs = let rev = "cd63096d6d887d689543a0b97743d28995bc9bc3";
+  in builtins.fetchTarball {
+    url = "https://github.com/NixOS/nixpkgs/archive/${rev}.tar.gz";
+    sha256 = "1wg61h4gndm3vcprdcg7rc4s1v3jkm5xd7lw8r2f67w502y94gcy";
+  };
+in import "${nixpkgs}/nixos" {
+  system = "x86_64-linux";
+
+  configuration = { config, pkgs, ... }: {
+    imports = [ "${nixpkgs}/nixos/modules/virtualisation/amazon-image.nix" ];
+
+    nix = {
+      binaryCaches =
+        [ "https://cache.nixos.org" "https://hercules-ci.cachix.org" ];
+      binaryCachePublicKeys = [
+        "hercules-ci.cachix.org-1:ZZeDl9Va+xe9j+KqdzoBZMFJHVQ42Uu/c/1/KMC5Lw0="
+      ];
+    };
+
+    ec2.hvm = true;
+    networking.hostName = hostName;
+
+    services.hercules-ci-agent.enable = true;
+    services.hercules-ci-agent.concurrentTasks = 4;
+
+    # The NixOS module has to be conservative with system-level settings,
+    # because they potentially affect other services.
+    # The hercules_ci_agent_nixos module does not have this design goal,
+    # so it can patch Nix automatically if necessary.
+    services.hercules-ci-agent.patchNix = true;
+
+    systemd.services.install-secrets = {
+      enable = true;
+      before = [ "hercules-ci-agent.service" ];
+      wantedBy = [ "hercules-ci-agent.service" ];
+      script = ''
+        install --directory \
+                --owner hercules-ci-agent \
+                --group nogroup \
+                --mode 0700 \
+                /var/lib/hercules-ci-agent/secrets \
+                ;
+        install --mode 0400 \
+                --owner hercules-ci-agent \
+                /var/keys/cluster_join_token \
+                /var/lib/hercules-ci-agent/secrets/cluster-join-token.key \
+                ;
+        install --mode 0400 \
+                --owner hercules-ci-agent \
+                /var/keys/binary_caches_json \
+                /var/lib/hercules-ci-agent/secrets/binary-caches.json \
+                ;
+      '';
+      serviceConfig.Type = "oneshot";
+    };
+
+    environment.systemPackages = [ pkgs.cloud-utils ];
+
+    # Limit journal size
+    services.journald.extraConfig = ''
+      SystemMaxUse=1024M
+    '';
+  };
+}

terraform/ci-runner.nix

@@ -0,0 +1,82 @@
+{ config, lib, pkgs, ... }:
+
+let
+  inherit (builtins) map filter getEnv replaceStrings toJSON toString;
+  inherit (lib) mkMerge concatStringsSep imap0;
+  terraform_nixos_repo =
+    "https://github.com/numtide/terraform-deploy-nixos-flakes.git";
+  terraform_nixos_ref = "b4093274bb1f0ae833c2e02298f5f032691601ac";
+  terraform_nixos = "git::${terraform_nixos_repo}?ref=${terraform_nixos_ref}";
+in {
+  options.servers.ci = with lib;
+    with types; {
+      subnet = mkOption { type = str; };
+      availability_zone = mkOption { type = str; };
+      instance_type = mkOption {
+        type = str;
+        default = "t3.small";
+      };
+      instance_tags = mkOption {
+        type = attrsOf str;
+        default = { };
+      };
+    };
+
+  config.module = {
+    deploy_ci_runner = {
+      source = terraform_nixos;
+      flake = "..";
+      flake_host = "dailp-ci-runner";
+      target_user = "root";
+      target_host = "\${aws_instance.ci_runner.public_ip}";
+      ssh_agent = false;
+      ssh_private_key = "\${var.aws_ssh_key}";
+      keys = {
+        cluster_join_token = "\${var.cluster_join_token}";
+        binary_caches_json = builtins.toJSON { };
+      };
+    };
+  };
+
+  config.resource = {
+    aws_instance.ci_runner = {
+      lifecycle.prevent_destroy = true;
+      ami = "\${module.nixos_image.ami}";
+      # We don't need many resources for CI.
+      instance_type = config.servers.ci.instance_type;
+      availability_zone = config.servers.ci.availability_zone;
+      key_name = "dailp-deployment-terraform";
+      # We only really need space for the nix store and caches.
+      root_block_device = {
+        volume_size = 16;
+        volume_type = "gp3";
+      };
+      tags = {
+        Name = "dailp-ci-runner";
+      } // config.setup.global_tags // config.servers.ci.instance_tags;
+      network_interface = {
+        device_index = 0;
+        network_interface_id = "\${aws_network_interface.ci_runner.id}";
+      };
+      iam_instance_profile = "\${aws_iam_instance_profile.ci_runner.id}";
+    };
+
+    aws_iam_instance_profile.ci_runner = {
+      path = "/";
+      # Use the MongoDB node role for now because it already configures VPC access.
+      role = "\${aws_iam_role.mongodb_node.id}";
+    };
+
+    aws_network_interface.ci_runner = {
+      subnet_id = config.servers.ci.subnet;
+      description = "Network interface for DAILP CI/CD";
+      # Grants SSH access under NEU VPN.
+      security_groups = [ "\${aws_security_group.nixos_test.id}" ];
+      source_dest_check = true;
+      tags = {
+        Network = "Public";
+        Name = "dailp-ci-runner";
+      };
+    };
+  };
+}

terraform/database-nixos.nix

@@ -5,6 +5,8 @@ let
   inherit (lib) mkMerge concatStringsSep imap0;
   terraform_nixos_repo = "https://github.com/tomferon/terraform-nixos.git";
   terraform_nixos_ref = "caa6191b952f8c92a097a67fc21200e2927e3d10";
+  terraform_nixos =
+    "git::${terraform_nixos_repo}//deploy_nixos?ref=${terraform_nixos_ref}";
   toKebabCase = s: replaceStrings [ "_" ] [ "-" ] s;
 in {
   options.servers.mongodb = with lib;
@@ -254,14 +256,13 @@ in {
         secondaries = if node.primary then "[${secondariesStr}]" else "null";
       in {
         "deploy_${name}" = {
-          source =
-            "git::${terraform_nixos_repo}//deploy_nixos?ref=${terraform_nixos_ref}";
+          source = terraform_nixos;
           nixos_config = toString ./mongodb-configuration.nix;
           hermetic = true;
           target_user = "root";
           target_host = "\${aws_instance.${name}.public_ip}";
           ssh_agent = false;
-          ssh_private_key = getEnv "AWS_SSH_KEY";
+          ssh_private_key = "\${var.aws_ssh_key}";
           arguments = mkMerge ([{
             hostName = config.resource.aws_instance."${name}".tags.Name;
             primaryAddress = primaryIp;

terraform/main.nix

@@ -9,12 +9,7 @@
 #   with Terraform property names.
 # - Functions should be in "camelCase" because that aligns with Nix standard practice.
 # - File names in this folder should be "kebab-case.nix"
-let
-  inherit (builtins) getEnv;
-  stage_var = getEnv "TF_STAGE";
-  # Default to the 'dev' environment unless specified.
-  stage = if stage_var == "" then "dev" else stage_var;
-in {
+{
   imports = [
     ./bootstrap.nix
     ./functions.nix
@@ -23,11 +18,32 @@ in {
     ./website.nix
     ./nu-tags.nix
     ./database-sql.nix
+    ./ci-runner.nix
   ];
 
+  variable = let
+    requiredString = { type = "string"; };
+    sensitiveString = requiredString // { sensitive = true; };
+  in {
+    aws_vpc_id = requiredString;
+    aws_subnet_primary = requiredString;
+    aws_subnet_secondary0 = requiredString;
+    aws_subnet_secondary1 = requiredString;
+    aws_zone_primary = requiredString;
+    aws_zone_secondary0 = requiredString;
+    aws_zone_secondary1 = requiredString;
+    git_repository_url = requiredString;
+
+    # Sensitive secret input variables
+    database_password = sensitiveString;
+    github_oauth_token = sensitiveString;
+    aws_ssh_key = sensitiveString;
+    cluster_join_token = sensitiveString;
+  };
+
   # Gives all modules access to which stage we're deploying to, while also
   # verifying that its one of the stages we actually use.
-  setup.stage = stage;
+  setup.stage = builtins.getEnv "TF_VAR_deployment_stage";
 
   terraform.required_providers.aws = {
     source = "hashicorp/aws";
@@ -48,11 +64,11 @@ in {
       bucket = "dailp-${config.setup.stage}-terraform-state-bucket";
       table = "dailp-${config.setup.stage}-terraform-state-locks";
     };
-    vpc = getEnv "AWS_VPC_ID";
+    vpc = "\${var.aws_vpc_id}";
     subnets = {
-      primary = getEnv "AWS_SUBNET_PRIMARY";
-      secondary = getEnv "AWS_SUBNET_SECONDARY0";
-      tertiary = getEnv "AWS_SUBNET_SECONDARY1";
+      primary = "\${var.aws_subnet_primary}";
+      secondary = "\${var.aws_subnet_secondary0}";
+      tertiary = "\${var.aws_subnet_secondary1}";
     };
   };
 
@@ -91,28 +107,33 @@ in {
     }];
   };
 
+  servers.ci = {
+    availability_zone = "\${var.aws_zone_primary}";
+    subnet = config.setup.subnets.primary;
+  };
+
   servers.database = {
-    password = getEnv "DATABASE_PASSWORD";
-    availability_zone = getEnv "AWS_ZONE_PRIMARY";
+    password = "\${var.database_password}";
+    availability_zone = "\${var.aws_zone_primary}";
     security_group_ids = [ "\${aws_security_group.nixos_test.id}" ];
   };
 
   servers.mongodb.nodes = [
     {
       primary = true;
-      availability_zone = getEnv "AWS_ZONE_PRIMARY";
+      availability_zone = "\${var.aws_zone_primary}";
       name = "mongodb_primary";
       subnet_id = config.setup.subnets.primary;
     }
     {
       primary = false;
-      availability_zone = getEnv "AWS_ZONE_SECONDARY0";
+      availability_zone = "\${var.aws_zone_secondary0}";
       name = "mongodb_secondary0";
       subnet_id = config.setup.subnets.secondary;
     }
     {
       primary = false;
-      availability_zone = getEnv "AWS_ZONE_SECONDARY1";
+      availability_zone = "\${var.aws_zone_secondary1}";
       name = "mongodb_secondary1";
       subnet_id = config.setup.subnets.tertiary;
     }

terraform/nu-tags.nix

@@ -20,6 +20,11 @@
     "nu:os" = "nixos";
     "nu:backups" = "no";
   };
+  servers.ci.instance_tags = {
+    "nu:function" = "ci";
+    "nu:os" = "nixos";
+    "nu:backups" = "no";
+  };
   servers.mongodb.storage_tags = {
     "nu:function" = "database";
     "nu:backups" = "no";

terraform/website.nix

@@ -33,8 +33,8 @@ with builtins; {
         tags = config.setup.global_tags;
         description =
           "Digital Archive of American Indian Languages Preservation and Perseverance";
-        repository = lib.toLower (getEnv "GIT_REPOSITORY_URL");
-        oauth_token = getEnv "OAUTH_TOKEN";
+        repository = "\${var.git_repository_url}";
+        oauth_token = "\${var.github_oauth_token}";
         iam_service_role_arn = "\${aws_iam_role.amplify_role.arn}";
         custom_rule = [
           {