Use ghcr.io to store closure

.github/workflows/hello.yml

@@ -0,0 +1,89 @@
+name: Test that we can build the hello world example form hackage
+on:
+  push:
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: devx {0}
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - x86_64-darwin
+          - x86_64-linux
+          # Skipping because we do not have runners for these set up.
+          #- aarch64-darwin
+          #- aarch64-linux
+        compiler-nix-name:
+          - ghc8107
+          - ghc927
+          - ghc961
+        target-platform:
+          - ""
+          - "-static"
+          - "-js"
+          # Comming soon.
+          #- "-windows"
+        variant:
+          - ""
+          - "-minimal"
+        iog:
+          - ""
+          - "-iog"
+        exclude:
+          # Just cross compiling javascript with ghc 9.6.1 for now
+          - compiler-nix-name: ghc8107
+            target-platform: "-js"
+          - compiler-nix-name: ghc927
+            target-platform: "-js"
+          # Static builds not working for darwin yet
+          - platform: x86_64-darwin
+            target-platform: "-static"
+          # Static tools not working right now (so just building "-static-minimal" for now)
+          - target-platform: "-static"
+            variant: ""
+          # hlint and hls not working on ghc 9.6.1 yet
+          - target-platform: ghc961
+            variant: ""
+          # Windows cross compilation only works on x86_64 right now.
+          - platform: aarch64-darwin
+            target-platform: "-windows"
+          - platform: aarch64-linux
+            target-platform: "-windows"
+
+    runs-on: ${{ matrix.platform == 'x86_64-linux' && 'ubuntu-latest' || 'macos-latest' }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Install Nix with good defaults
+        uses: cachix/install-nix-action@v17
+        with:
+          extra_nix_config: |
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=
+            substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: Install jq
+        shell: bash
+        run: |
+          curl -L https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -o jq
+          chmod +x jq
+      - name: Download and import closure
+        shell: bash
+        run: |
+          ./fetch-docker.sh input-output-hk/devx ${{ matrix.platform }}.${{ matrix.compiler-nix-name }}${{ matrix.target-platform }}${{ matrix.variant }}${{ matrix.iog }}-env | zstd -d | nix-store --import | tee store-paths.txt
+          sudo ln -s $(tail -n 1 store-paths.txt) /usr/local/bin/devx-env.sh
+          sudo cp extra/devx /usr/local/bin
+      - name: cabal update
+        run: cabal update
+      - name: unapck hello
+        run: cabal unpack hello
+      - name: build hello
+        working-directory: hello-1.0.0.2
+        run: cabal build
+      - name: test hello
+        working-directory: hello-1.0.0.2
+        run: cabal test

.github/workflows/main.yml

@@ -0,0 +1,92 @@
+name: Prebuild DevX closures
+on:
+  push:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - x86_64-darwin
+          - x86_64-linux
+          - aarch64-darwin
+          # Currently broken
+          #- aarch64-linux
+        compiler-nix-name:
+          - ghc8107
+          - ghc927
+          - ghc961
+        target-platform:
+          - ""
+          - "-static"
+          - "-js"
+          # Coming soon.
+          #- "-windows"
+        variant:
+          - ""
+          - "-minimal"
+        iog:
+          - ""
+          - "-iog"
+        exclude:
+          # Just cross compiling javascript with ghc 9.6.1 for now
+          - compiler-nix-name: ghc8107
+            target-platform: "-js"
+          - compiler-nix-name: ghc927
+            target-platform: "-js"
+          # Static builds not working for darwin yet
+          - platform: x86_64-darwin
+            target-platform: "-static"
+          # Static tools not working right now (so just building "-static-minimal" for now)
+          - target-platform: "-static"
+            variant: ""
+          # hlint and hls not working on ghc 9.6.1 yet
+          - target-platform: ghc961
+            variant: ""
+          # Windows cross compilation only works on x86_64 right now.
+          - platform: aarch64-darwin
+            target-platform: "-windows"
+          - platform: aarch64-linux
+            target-platform: "-windows"
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Install Nix with good defaults
+        uses: cachix/install-nix-action@v17
+        with:
+          extra_nix_config: |
+            trusted-public-keys = cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= loony-tools:pr9m4BkM/5/eSTZlkQyRt57Jz7OMBxNSUiMC4FkcNfk=
+            substituters = https://cache.iog.io/ https://cache.zw3rk.com/ https://cache.nixos.org/
+          nix_path: nixpkgs=channel:nixos-unstable
+      - name: Checkout repository
+        uses: actions/checkout@v3
+      - name: Log in to the Container registry
+        uses: docker/login-action@v1
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Work around issue fetching cabal revision files
+        run: |
+          # This will cause the revised `.cabal` files used whan building alex
+          # for building GHC to be fetched using the `x86_64-linux` `.drv` for `fetchurl`.
+          # Later when we want the non `x86_64-linux` version later it will find the
+          # fixed output of the derivation is already in the `/nix/store` and will
+          # not try to fetch it using the platform we do not have a builder for.
+          if [[ "${{ matrix.platform }}" != "x86_64-linux" && "${{ matrix.target-platform }}" = "-js" ]]; then
+            nix build ".#hydraJobs.x86_64-linux.${{ matrix.compiler-nix-name }}-js-minimal" --show-trace
+          fi
+      - name: Compute and upload closure and developer environment to ghcr.io
+        env:
+          NIX_STORE_SECRET_KEY: ${{ secrets.SECRET_KEY }}
+          DEV_SHELL: ${{ matrix.platform }}.${{ matrix.compiler-nix-name }}${{ matrix.target-platform }}${{ matrix.variant }}${{ matrix.iog }}-env
+        run: ./extra/ghcr-upload.sh

cross-js.nix

@@ -1,6 +1,6 @@
-{ pkgs, compiler, compiler-nix-name, withHLS ? true, withHlint ? true, withIOG ? true  }:
+{ pkgs, compiler, compiler-nix-name, toolsModule, withHLS ? true, withHlint ? true, withIOG ? true  }:
 let tool-version-map = import ./tool-map.nix;
-    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name (tool-version-map compiler-nix-name tool-name);
+    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name [(tool-version-map compiler-nix-name tool-name) toolsModule];
     cabal-install = tool "cabal";
     # add a trace helper. This will trace a message about disabling a component despite requesting it, if it's not supported in that compiler.
     compiler-not-in = compiler-list: name: (if __elem compiler-nix-name compiler-list then __trace "No ${name}. Not yet compatible with ${compiler-nix-name}" false else true);    
@@ -20,7 +20,7 @@ let tool-version-map = import ./tool-map.nix;
                 "$@" \
                 $NIX_CABAL_FLAGS 
             ;;
-            clean)
+            clean|unpack)
             cabal "$@"
             ;;
             *)
@@ -29,6 +29,12 @@ let tool-version-map = import ./tool-map.nix;
         esac        
         '';
     };    
+    wrapped-hsc2hs = pkgs.pkgsBuildBuild.writeShellApplication {
+        name = "${compiler.targetPrefix}hsc2hs";
+        text = ''
+          ${compiler}/bin/${compiler.targetPrefix}hsc2hs --cross-compile "$@"
+        '';
+    };
 in
 pkgs.mkShell ({
     # Note [cabal override]:
@@ -43,10 +49,10 @@ pkgs.mkShell ({
     # open source, and licenses accordingly.  Otherwise we'd have to link gmp
     # dynamically.  This requirement will be gone with gmp-bignum.
     #
-    NIX_CABAL_FLAGS = pkgs.lib.optionals pkgs.stdenv.hostPlatform.isGhcjs [
-    "--with-ghc=javascript-unknown-linux-ghcjs-ghc"
-    "--with-ghc-pkg=javascript-unknown-linux-ghcjs-pkg"
-    "--with-hsc2hs=javascript-unknown-linux-ghcjs-hsc2hs"
+    NIX_CABAL_FLAGS = [
+    "--with-ghc=javascript-unknown-ghcjs-ghc"
+    "--with-ghc-pkg=javascript-unknown-ghcjs-ghc-pkg"
+    "--with-hsc2hs=javascript-unknown-ghcjs-hsc2hs"
     # ensure that the linker knows we want a static build product
     # "--enable-executable-static"
     ];
@@ -69,9 +75,11 @@ pkgs.mkShell ({
     '';
     buildInputs = [];
 
-    nativeBuildInputs = [ wrapped-cabal compiler ] ++ (with pkgs; [
+    nativeBuildInputs = [ wrapped-hsc2hs wrapped-cabal compiler ] ++ (with pkgs; [
         haskell-nix.cabal-install.${compiler-nix-name}
         pkgconfig
+        (tool "happy")
+        (tool "alex")
         stdenv.cc.cc.lib ]) ++ (with pkgs.buildPackages; [
     ])
     ++ pkgs.lib.optional (withHLS && (compiler-not-in ["ghc961"] "Haskell Language Server")) (tool "haskell-language-server")

dynamic.nix

@@ -1,7 +1,7 @@
 # define a development shell for dynamically linked applications (default)
-{ pkgs, compiler, compiler-nix-name, withHLS ? true, withHlint ? true, withIOG ? true }:
+{ pkgs, compiler, compiler-nix-name, toolsModule, withHLS ? true, withHlint ? true, withIOG ? true }:
 let tool-version-map = import ./tool-map.nix;
-    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name (tool-version-map compiler-nix-name tool-name);
+    tool = tool-name: pkgs.haskell-nix.tool compiler-nix-name tool-name [(tool-version-map compiler-nix-name tool-name) toolsModule];
     cabal-install = tool "cabal";
     # add a trace helper. This will trace a message about disabling a component despite requesting it, if it's not supported in that compiler.
     compiler-not-in = compiler-list: name: (if __elem compiler-nix-name compiler-list then __trace "No ${name}. Not yet compatible with ${compiler-nix-name}" false else true);
@@ -18,7 +18,7 @@ let tool-version-map = import ./tool-map.nix;
         case "$1" in
             build) cabal "$@"
             ;;
-            clean) cabal "$@"
+            clean|unpack) cabal "$@"
             ;;
             *) cabal "$@"
             ;;
@@ -33,6 +33,8 @@ pkgs.mkShell {
     shellHook = with pkgs; ''
         export PS1="\[\033[01;33m\][\w]$\[\033[00m\] "
         ${figlet}/bin/figlet -f rectangles 'IOG Haskell Shell'
+        export CABAL_DIR=$HOME/.cabal
+        echo "CABAL_DIR set to $CABAL_DIR"
     ''
     # this one is only needed on macOS right now, due to a bug in loading libcrypto.
     + lib.optionalString stdenv.hostPlatform.isMacOS
@@ -57,8 +59,8 @@ pkgs.mkShell {
         ]
         ++ pkgs.lib.optional pkgs.stdenv.hostPlatform.isLinux systemd
     )
-    ++ pkgs.lib.optional (withHLS && (compiler-not-in ["ghc961"] "Haskell Language Server")) (tool "haskell-language-server")
-    ++ pkgs.lib.optional (withHlint && (compiler-not-in ["ghc961"] "HLint")) (tool "hlint")
+    ++ pkgs.lib.optional (withHLS && (compiler-not-in (["ghc961"] ++ pkgs.lib.optional (pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) "ghc902") "Haskell Language Server")) (tool "haskell-language-server")
+    ++ pkgs.lib.optional (withHlint && (compiler-not-in (["ghc961"] ++ pkgs.lib.optional (pkgs.stdenv.hostPlatform.isDarwin && pkgs.stdenv.hostPlatform.isAarch64) "ghc902") "HLint")) (tool "hlint")
     ++ pkgs.lib.optional withIOG
         (with pkgs; [ cddl cbor-diag ]
         ++ map pkgs.lib.getDev (with pkgs; [ libsodium-vrf secp256k1 R_4_1_3]))

extra/devx

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -euo pipefail
+
+source /usr/local/bin/devx-env.sh
+source "$1"

extra/ghcr-upload.sh

@@ -0,0 +1,7 @@
+#! /usr/bin/env nix-shell
+#! nix-shell -i bash -p zstd -p oras
+set -euox pipefail
+
+nix build ".#hydraJobs.${DEV_SHELL}" --show-trace
+nix-store --export $(nix-store -qR result) | zstd -z8T8 >${DEV_SHELL}
+oras push ghcr.io/input-output-hk/devx:${DEV_SHELL} ${DEV_SHELL}

fetch-docker.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+TOKEN=$(curl --silent https://ghcr.io/token\?scope\=repository:$1:pull | jq -r .token)
+
+BLOB=$(curl \
+--silent \
+--request 'GET' \
+--header "Authorization: Bearer $TOKEN" \
+--header "Accept: application/vnd.docker.distribution.manifest.list.v2+json" \
+--header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+--header "Accept: application/vnd.oci.image.manifest.v1+json" \
+"https://ghcr.io/v2/$1/manifests/$2" | tee manifest.json | jq -r '.layers[0].digest')
+
+curl \
+--location \
+--request GET \
+--header "Authorization: Bearer ${TOKEN}" \
+"https://ghcr.io/v2/$1/blobs/${BLOB}"