First version

.envrc

@@ -0,0 +1,5 @@
+eval "$(lorri direnv)"
+
+# Use system PKI
+unset SSL_CERT_FILE
+unset NIX_SSL_CERT_FILE

README.md

@@ -1,2 +1,11 @@
 # terraform-hercules-ci
 Terraform modules for Hercules CI
+
+In this repo:
+
+ - [hercules_ci_agent_nixos](./hercules_ci_agent_nixos)
+ - [hercules_ci_agent_aws](./hercules_ci_agent_aws)
+
+Related repos:
+
+ - [example AWS deployment](https://github.com/hercules-ci/example-deploy-agent-terraform-aws)

hercules_ci_agent_aws/README.md

@@ -0,0 +1,4 @@
+
+# hercules_ci_agent_aws module
+
+Deploys a basic EC2 agent.

hercules_ci_agent_aws/configuration-aws.nix

@@ -0,0 +1,3 @@
+{ ... }: {
+  imports = [ <nixpkgs/nixos/modules/virtualisation/amazon-image.nix> ];
+}

hercules_ci_agent_aws/main.tf

@@ -0,0 +1,57 @@
+
+module "nixos" {
+  source = "../hercules_ci_agent_nixos"
+  target_host = "${aws_instance.machine.public_ip}"
+  use_prebuilt = "${var.use_prebuilt}"
+  configs = [
+    "${path.module}/configuration-aws.nix",
+    "${var.configs}",
+  ]
+  cluster_join_token = "${var.cluster_join_token}"
+  binary_caches_json = "${var.binary_caches_json}"
+  triggers = {
+    machine_id = "${aws_instance.machine.id}"
+  }
+}
+
+module "nixos_image_1903" {
+  source = "git::https://github.com/tweag/terraform-nixos.git//aws_image_nixos?ref=d61e2a193620df13af5930be16f5b1f572f95ffa"
+  release = "19.03"
+}
+
+resource "aws_security_group" "ssh_and_egress" {
+  ingress {
+    # SSH for deployment
+    from_port   = 22
+    to_port     = 22
+    protocol    = "tcp"
+    cidr_blocks = [ "0.0.0.0/0" ] # TODO: restrict
+  }
+
+  egress {
+    from_port       = 0
+    to_port         = 0
+    protocol        = "-1"
+    cidr_blocks     = ["0.0.0.0/0"]
+  }
+}
+
+resource "aws_key_pair" "deployer" {
+  key_name   = "deployer-key-${sha256(var.public_key)}"
+  public_key = "${var.public_key}"
+}
+
+resource "aws_instance" "machine" {
+  ami           = "${module.nixos_image_1903.ami}"
+  instance_type = "${var.instance_type}"
+  security_groups = [ "${aws_security_group.ssh_and_egress.name}" ]
+  key_name = "${aws_key_pair.deployer.key_name}"
+
+  root_block_device {
+    volume_size = "${var.disk_size}" # GiB
+  }
+}
+
+output "public_dns" {
+  value = "${aws_instance.machine.public_dns}"
+}

hercules_ci_agent_aws/variables.tf

@@ -0,0 +1,38 @@
+
+variable "use_prebuilt" {
+  type = "string"
+  description = "Get agent binaries from the hercules-ci cache"
+  default = true
+}
+
+variable "configs" {
+  type = "list"
+  description = "Extra NixOS configuration modules to import"
+}
+
+variable "cluster_join_token" {
+  type = "string"
+  description = "The cluster join token contents. See https://docs.hercules-ci.com/hercules-ci/reference/agent-config/#clusterJoinTokenPath"
+}
+
+variable "binary_caches_json" {
+  type = "string"
+  description = "The binary-caches.json contents. See https://docs.hercules-ci.com/hercules-ci/reference/agent-config/#binaryCachesPath"
+}
+
+variable "public_key" {
+  type = "string"
+  description = "An SSH public key. This corresponding private key must be loaded into the SSH agent, for example: `ssh-add ./id_rsa`."
+}
+
+variable "instance_type" {
+  type = "string"
+  description = "Amazon EC2 instance type"
+  default = "t3.medium"
+}
+
+variable "disk_size" {
+  type = "string"
+  description = "Size of the filesystem in GiB"
+  default = 100
+}

hercules_ci_agent_nixos/README.md

@@ -0,0 +1,5 @@
+# hercules_ci_agent_nixos Terraform module
+
+This module deploys a suitable NixOS configuration onto a system that runs NixOS.
+
+

hercules_ci_agent_nixos/configuration.nix

@@ -0,0 +1,34 @@
+{ pkgs, lib, ... }:
+let
+  sources = import ./sources.nix;
+in
+{
+  imports = [ (sources.hercules-ci-agent + "/module.nix") ];
+  services.hercules-ci-agent.enable = true;
+
+  systemd.services.install-secrets = {
+    enable = true;
+    before = [ "hercules-ci-agent.service" ];
+    wantedBy = [ "hercules-ci-agent.service" ];
+    script = ''
+      install --directory \
+              --owner hercules-ci-agent \
+              --group nogroup \
+              --mode 0700 \
+              /var/lib/hercules-ci-agent/secrets \
+              ;
+      install --mode 0400 \
+              --owner hercules-ci-agent \
+              /var/keys/cluster_join_token \
+              /var/lib/hercules-ci-agent/secrets/cluster-join-token.key \
+              ;
+      install --mode 0400 \
+              --owner hercules-ci-agent \
+              /var/keys/binary_caches_json \
+              /var/lib/hercules-ci-agent/secrets/binary-caches.json \
+              ;
+    '';
+    serviceConfig.Type = "oneshot";
+  };
+
+}

hercules_ci_agent_nixos/hercules-ci-cache.nix

@@ -0,0 +1,16 @@
+/*
+
+  Adds the hercules-ci cache to use binaries instead of rebuilding from source.
+
+ */
+{
+  nix = {
+    binaryCaches = [
+      "https://cache.nixos.org"
+      "https://hercules-ci.cachix.org"
+    ];
+    binaryCachePublicKeys = [
+      "hercules-ci.cachix.org-1:ZZeDl9Va+xe9j+KqdzoBZMFJHVQ42Uu/c/1/KMC5Lw0="
+    ];
+  };
+}

hercules_ci_agent_nixos/main.tf

@@ -0,0 +1,42 @@
+
+locals {
+  configs = [
+    "${path.module}/configuration.nix",
+    "${var.use_prebuilt ? "${path.module}/hercules-ci-cache.nix" : "" }",
+    "${var.configs}"
+  ]
+}
+
+data "external" "nixpkgs_src" {
+  program = [
+    "/usr/bin/env",
+    "NIX_PATH=",
+    "nix-instantiate",
+    "--eval",
+    "--expr",
+    "--json",
+    "--read-write-mode",
+    "--strict",
+    "{ src }: let sources = import (src + \"/nix/sources.nix\"); in { nixpkgs = builtins.readFile (builtins.toFile \"nixpkgs\" sources.\"nixos-19.03\".outPath); }",
+    "--arg",
+    "src",
+    "~/h/hercules-ci-agent",
+  ]
+}
+
+module "deploy_nixos" {
+  source = "git::https://github.com/tweag/terraform-nixos.git//deploy_nixos?ref=d61e2a193620df13af5930be16f5b1f572f95ffa"
+
+  config = "{ pkgs, lib, ... }: { imports = [ (/. + ''${join("'') (/. + ''",compact(flatten(local.configs)))}'') ]; }"
+
+  target_user = "root"
+  target_host = "${var.target_host}"
+  NIX_PATH = "nixpkgs=${data.external.nixpkgs_src.result["nixpkgs"]}"
+
+  triggers = "${var.triggers}"
+
+  keys = {
+    cluster_join_token = "${var.cluster_join_token}"
+    binary_caches_json = "${var.binary_caches_json}"
+  }
+}

hercules_ci_agent_nixos/sources.json

@@ -0,0 +1,50 @@
+{
+    "hercules-ci-agent": {
+        "branch": "master",
+        "description": "https://hercules-ci.com build agent",
+        "homepage": "",
+        "owner": "hercules-ci",
+        "repo": "hercules-ci-agent",
+        "rev": "be768740c171a9fd7b6c3ff82638d0a4fb66aa8b",
+        "sha256": "1l1vdplvchf5fxdc9hf33vhy8s2s7773mf2c6231v6612m0yqs7p",
+        "type": "tarball",
+        "url": "https://github.com/hercules-ci/hercules-ci-agent/archive/be768740c171a9fd7b6c3ff82638d0a4fb66aa8b.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "niv": {
+        "branch": "master",
+        "description": "Easy dependency management for Nix projects",
+        "homepage": "https://github.com/nmattia/niv",
+        "owner": "nmattia",
+        "repo": "niv",
+        "rev": "1dd094156b249586b66c16200ecfd365c7428dc0",
+        "sha256": "1b2vjnn8iac5iiqszjc2v1s1ygh0yri998c0k3s4x4kn0dsqik21",
+        "type": "tarball",
+        "url": "https://github.com/nmattia/niv/archive/1dd094156b249586b66c16200ecfd365c7428dc0.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "nixpkgs": {
+        "branch": "nixos-19.03",
+        "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to",
+        "homepage": "https://github.com/NixOS/nixpkgs",
+        "owner": "NixOS",
+        "repo": "nixpkgs-channels",
+        "rev": "021d733ea3f87b8c9232020b4e606d08eaca160b",
+        "sha256": "13600nzrakvg2hsfg5yr7x0jp9m762nvjyddf07q60d3m7vx9jxy",
+        "type": "tarball",
+        "url": "https://github.com/NixOS/nixpkgs-channels/archive/021d733ea3f87b8c9232020b4e606d08eaca160b.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    },
+    "terraform-nixos": {
+        "branch": "master",
+        "description": "A set of Terraform modules that are designed to deploy NixOS",
+        "homepage": null,
+        "owner": "tweag",
+        "repo": "terraform-nixos",
+        "rev": "d61e2a193620df13af5930be16f5b1f572f95ffa",
+        "sha256": "0ybgvkzsl6n82wka1zdzfmkyi7ihihxk7vl99r8pwll4w0xw2ypm",
+        "type": "tarball",
+        "url": "https://github.com/tweag/terraform-nixos/archive/d61e2a193620df13af5930be16f5b1f572f95ffa.tar.gz",
+        "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
+    }
+}

hercules_ci_agent_nixos/sources.nix

@@ -0,0 +1,93 @@
+# This file has been generated by Niv.
+
+# A record, from name to path, of the third-party packages
+with rec
+{
+  pkgs =
+    if hasNixpkgsPath
+    then
+        if hasThisAsNixpkgsPath
+        then import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {}
+        else import <nixpkgs> {}
+    else
+        import (builtins_fetchTarball { inherit (sources_nixpkgs) url sha256; }) {};
+
+  sources_nixpkgs =
+    if builtins.hasAttr "nixpkgs" sources
+    then sources.nixpkgs
+    else abort
+    ''
+        Please specify either <nixpkgs> (through -I or NIX_PATH=nixpkgs=...) or
+        add a package called "nixpkgs" to your sources.json.
+    '';
+
+  # fetchTarball version that is compatible between all the versions of Nix
+  builtins_fetchTarball =
+      { url, sha256 }@attrs:
+      let
+        inherit (builtins) lessThan nixVersion fetchTarball;
+      in
+        if lessThan nixVersion "1.12" then
+          fetchTarball { inherit url; }
+        else
+          fetchTarball attrs;
+
+  # fetchurl version that is compatible between all the versions of Nix
+  builtins_fetchurl =
+      { url, sha256 }@attrs:
+      let
+        inherit (builtins) lessThan nixVersion fetchurl;
+      in
+        if lessThan nixVersion "1.12" then
+          fetchurl { inherit url; }
+        else
+          fetchurl attrs;
+
+  # A wrapper around pkgs.fetchzip that has inspectable arguments,
+  # annoyingly this means we have to specify them
+  fetchzip = { url, sha256 }@attrs: pkgs.fetchzip attrs;
+
+  # A wrapper around pkgs.fetchurl that has inspectable arguments,
+  # annoyingly this means we have to specify them
+  fetchurl = { url, sha256 }@attrs: pkgs.fetchurl attrs;
+
+  hasNixpkgsPath = (builtins.tryEval <nixpkgs>).success;
+  hasThisAsNixpkgsPath =
+    (builtins.tryEval <nixpkgs>).success && <nixpkgs> == ./.;
+
+  sources = builtins.fromJSON (builtins.readFile ./sources.json);
+
+  mapAttrs = builtins.mapAttrs or
+    (f: set: with builtins;
+      listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)));
+
+  # borrowed from nixpkgs
+  functionArgs = f: f.__functionArgs or (builtins.functionArgs f);
+  callFunctionWith = autoArgs: f: args:
+    let auto = builtins.intersectAttrs (functionArgs f) autoArgs;
+    in f (auto // args);
+
+  getFetcher = spec:
+    let fetcherName =
+      if builtins.hasAttr "type" spec
+      then builtins.getAttr "type" spec
+      else "builtin-tarball";
+    in builtins.getAttr fetcherName {
+      "tarball" = fetchzip;
+      "builtin-tarball" = builtins_fetchTarball;
+      "file" = fetchurl;
+      "builtin-url" = builtins_fetchurl;
+    };
+};
+# NOTE: spec must _not_ have an "outPath" attribute
+mapAttrs (_: spec:
+  if builtins.hasAttr "outPath" spec
+  then abort
+    "The values in sources.json should not have an 'outPath' attribute"
+  else
+    if builtins.hasAttr "url" spec && builtins.hasAttr "sha256" spec
+    then
+      spec //
+      { outPath = callFunctionWith spec (getFetcher spec) { }; }
+    else spec
+  ) sources