Move channel generation to bastion

Channel redirects are now stored in S3. Fixes #72. Fixes #73.
NixOS · Jan 24, 2019 · 3893d59 · 3893d59
1 parent 66fd86d
commit 3893d59
Show file tree

Hide file tree

Showing 5 changed files with 23 additions and 170 deletions.
diff --git a/bastion/network.nix b/bastion/network.nix
@@ -83,6 +83,12 @@ in
       vpc = true;
     };
 
+  resources.ebsVolumes.scratch =
+    { tags.Name = "Scratch space for the channel generator";
+      inherit region zone accessKeyId;
+      size = 64;
+    };
+
   bastion =
     { config, pkgs, resources, ... }:
 
@@ -104,6 +110,7 @@ in
       imports =
         [ ../modules/common.nix
           ../modules/tarball-mirror.nix
+          ../modules/hydra-mirror.nix
         ];
 
       users.extraUsers.tarball-mirror.openssh.authorizedKeys.keys = [ sshKeys.eelco ];
@@ -130,5 +137,14 @@ in
         ''
           AcceptEnv AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY FASTLY_API_KEY
         '';
+
+      fileSystems."/scratch" =
+        { autoFormat = true;
+          fsType = "ext4";
+          device = "/dev/xvdh";
+          ec2.disk = resources.ebsVolumes.scratch;
+        };
+
+      systemd.tmpfiles.rules = [ "d /scratch/hydra-mirror 0755 hydra-mirror users 10d" ];
     };
 }
diff --git a/modules/common.nix b/modules/common.nix
@@ -12,7 +12,7 @@ with lib;
 
   nix.useSandbox = true;
   nix.buildCores = 0;
-  nix.nixPath = [ "nixpkgs=channel:nixos-18.03-small" ];
+  nix.nixPath = [ "nixpkgs=channel:nixos-18.09-small" ];
 
   environment.systemPackages =
     [ pkgs.emacs

diff --git a/nixos-org/hydra-mirror.nix → modules/hydra-mirror.nix b/nixos-org/hydra-mirror.nix → modules/hydra-mirror.nix
@@ -26,6 +26,7 @@ let
               exec mirror-nixos-branch ${channelName} https://hydra.nixos.org/job/${mainJob}/latest-finished
             ''; # */
           serviceConfig.User = "hydra-mirror";
+          environment.TMPDIR = "/scratch/hydra-mirror";
         };
     };
 
@@ -55,5 +56,4 @@ in
       (makeUpdateChannel "nixpkgs-17.09-darwin" "nixpkgs/nixpkgs-17.09-darwin/darwin-tested")
       (makeUpdateChannel "nixpkgs-unstable" "nixpkgs/trunk/unstable")
     ];
-
 }
diff --git a/nixos-org/network.nix b/nixos-org/network.nix
@@ -13,13 +13,6 @@ in
       size = 1024;
     };
 
-  # FIXME: remove
-  resources.ebsVolumes.data =
-    { tags.Name = "Misc. NixOS.org data";
-      inherit region zone accessKeyId;
-      size = 10;
-    };
-
   resources.ebsVolumes.data-new =
     { tags.Name = "Misc. NixOS.org data";
       inherit region zone accessKeyId;
@@ -35,166 +28,6 @@ in
     { inherit region accessKeyId;
     };
 
-  resources.s3Buckets.nixpkgs-tarballs =
-    { config, ... }:
-    let
-      allowUser = name: arn:
-        [
-          {
-            Sid = "${name}AllowUpload";
-            Effect = "Allow";
-            Principal.AWS = arn;
-            Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
-            Resource = [ "${config.arn}/*" ];
-          }
-          {
-            Sid = "${name}AllowUpload2";
-            Effect = "Allow";
-            Principal.AWS = arn;
-            Action = [ "s3:ListBucket" ];
-            Resource = [ "${config.arn}" ];
-          }
-        ];
-    in
-    { inherit region accessKeyId;
-      name = "nixpkgs-tarballs";
-      # All files are readable but not listable.
-      # The s3-upload-tarballs user can upload files.
-      policy = builtins.toJSON
-        { Version = "2008-10-17";
-          Statement =
-            [
-              {
-                Sid = "AllowPublicRead";
-                Effect = "Allow";
-                Principal.AWS = "*";
-                Action = [ "s3:GetObject" ];
-                Resource = [ "${config.arn}/*" ];
-              }
-              {
-                Sid = "AllowUpload";
-                Effect = "Allow";
-                Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-tarballs";
-                Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
-                Resource = [ "${config.arn}/*" ];
-              }
-              {
-                Sid = "AllowUpload2";
-                Effect = "Allow";
-                Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-tarballs";
-                Action = [ "s3:ListBucket" ];
-                Resource = [ "${config.arn}" ];
-              }
-            ]
-            ++ allowUser "Copumpkin" "arn:aws:iam::390897850978:root"
-            ++ allowUser "Shlevy" "arn:aws:iam::976576280863:user/shlevy"
-            ++ allowUser "Daiderd" "arn:aws:iam::014292808257:user/lnl7"
-            ;
-        };
-      website.enabled = true;
-    };
-
-  resources.s3Buckets.nix-cache =
-    { config, ... }:
-    { inherit accessKeyId;
-      region = "us-east-1";
-      name = "nix-cache";
-      policy = builtins.toJSON {
-        Version = "2008-10-17";
-        Statement =
-          [
-            { Sid = "AllowPublicRead";
-              Effect = "Allow";
-              Principal.AWS = "*";
-              Action = [ "s3:GetObject" ];
-              Resource = [ "${config.arn}/*" ];
-            }
-            { Sid = "AllowUploadDebuginfoWrite";
-              Effect = "Allow";
-              Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
-              Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
-              Resource = [ "${config.arn}/debuginfo/*" ];
-            }
-            { Sid = "AllowUploadDebuginfoRead";
-              Effect = "Allow";
-              Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
-              Action = [ "s3:GetObject" ];
-              Resource = [ "${config.arn}/*" ];
-            }
-            { Sid = "AllowUploadDebuginfoRead2";
-              Effect = "Allow";
-              Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
-              Action = [ "s3:ListBucket" "s3:GetBucketLocation" ];
-              Resource = [ "${config.arn}" ];
-            }
-          ];
-      };
-    };
-
-  /*
-  resources.s3Buckets.nix-test-cache =
-    { config, ... }:
-    { inherit region accessKeyId;
-      name = "nix-test-cache";
-      policy =
-        ''
-          {
-            "Version": "2008-10-17",
-            "Statement": [
-              {
-                "Sid": "AllowPublicRead",
-                "Effect": "Allow",
-                "Principal": {"AWS": "*"},
-                "Action": ["s3:GetObject"],
-                "Resource": ["${config.arn}/*"]
-              },
-              {
-                "Sid": "AllowPublicList",
-                "Effect": "Allow",
-                "Principal": {"AWS": "*"},
-                "Action": ["s3:ListBucket"],
-                "Resource": ["${config.arn}"]
-              }
-            ]
-          }
-        '';
-    };
-  */
-
-  resources.s3Buckets.nix-releases =
-    { config, ... }:
-    { inherit accessKeyId;
-      name = "nix-releases";
-      region = "eu-west-1";
-      policy = builtins.toJSON
-        { Version = "2008-10-17";
-          Statement =
-            [
-              {
-                Sid = "AllowPublicRead";
-                Effect = "Allow";
-                Principal.AWS = "*";
-                Action = [ "s3:GetObject" ];
-                Resource = [ "${config.arn}/*" ];
-              }
-              {
-                Sid = "AllowPublicList";
-                Effect = "Allow";
-                Principal.AWS = "*";
-                Action = [ "s3:ListBucket" ];
-                Resource = [ "${config.arn}" ];
-              }
-              {
-                Sid = "AllowUpload";
-                Effect = "Allow";
-                Principal.AWS = "arn:aws:iam::080433136561:user/s3-upload-releases";
-                Action = [ "s3:PutObject" "s3:PutObjectAcl" ];
-                Resource = [ "${config.arn}/*" ];
-              }
-            ];
-        };
-    };
-
   resources.vpc.nixos-org-vpc =
     {
       inherit region accessKeyId;
@@ -308,7 +141,7 @@ in
 
       system.stateVersion = "17.09";
 
-      imports = [ ./webserver.nix ./hydra-mirror.nix ];
+      imports = [ ./webserver.nix ];
     };
 
 }
diff --git a/terraform/releases.tf b/terraform/releases.tf
@@ -1,6 +1,10 @@
 resource "aws_s3_bucket" "releases" {
   bucket = "nix-releases"
 
+  website {
+    index_document = "index.html"
+  }
+
   cors_rule {
     allowed_headers = ["*"]
     allowed_methods = ["HEAD", "GET"]