Merge #2645

2645: Optimize Hydra eval r=rvl a=rvl

### Issue Number

ADP-921

### Overview

- Copies the fix from IntersectMBO/cardano-node#2315 to reduce maximum memory usage of cardano-wallet evaluations.
- The release jobset needed to be completely rearranged for the above fix.
- Change the hydra "required" job constituents to be a list of string job names (instead of the actual derivations). This is supposed to make the aggregate build step quicker.
- Add a github action to record build time metrics on a separate git branch.
- Fix a windows release build job filename
- Enable pretty-simple in `.ghci` config and nix-shell tweak.

### Comments

- [Jobset link](https://hydra.iohk.io/jobset/Cardano/cardano-wallet-pr-2645)


Co-authored-by: Rodney Lorrimar <rodney.lorrimar@iohk.io>

.github/workflows/ci-stats.yml

@@ -0,0 +1,52 @@
+name: Stats
+
+on:
+  status:
+  push:
+    branches:
+      - master
+      - rvl/adp-921/optimize-hydra-eval
+
+jobs:
+  stats:
+    env:
+      branch: ci-stats
+    if: ${{ (github.ref == 'refs/heads/master' && (github.event.state == 'success' || github.event.state == 'failure') && github.event.context == 'ci/hydra-build:required') || github.ref == 'refs/heads/rvl/adp-921/optimize-hydra-eval' }}
+    runs-on: ubuntu-20.04
+    steps:
+      - name: '❄ Wait for Hydra build'
+        uses: rvl/hydra-build-products-action@master
+        id: hydra
+        with:
+          hydra: https://hydra.iohk.io
+          statusName: ci/hydra-eval
+          jobs: required
+          requiredJob: required
+          project: Cardano
+          jobset: cardano-wallet
+          badge: "${{ github.event_name == 'status' && 'yes' || 'no' }}"
+
+      - name: '📥 Checkout Code'
+        uses: actions/checkout@v2
+
+      - name: '📥 Checkout Stats'
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ env.branch }}
+          fetch-depth: 1
+          path: _build
+
+      - name: '📘 Publish Stats'
+        if: ${{ github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rvl/adp-921/optimize-hydra-eval' }}
+        working-directory: _build
+        run: |
+          echo '${{ steps.hydra.outputs.evaluation }}' > eval.json
+          echo '${{ steps.hydra.outputs.builds }}' > builds.json
+          echo '${{ steps.hydra.outputs.timings }}' > timings.json
+          ../scripts/gh/update-stats.sh '${{ env.branch}}' eval.json builds.json timings.json
+
+      - name: '📛 Update Badge'
+        if: ${{ github.event_name == 'status' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/rvl/adp-921/optimize-hydra-eval') }}
+        working-directory: _build
+        run: |
+          ../scripts/gh/update-badge.sh '${{ env.branch}}' <(echo '${{ steps.hydra.outputs.badge }}')

.github/workflows/release.yml

@@ -38,7 +38,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: '📥 Checkout repository'
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
 
       - name: '📸 Build Documentation'
         run: |
@@ -47,7 +47,7 @@ jobs:
           else
             tag=""
           fi
-          ./scripts/update-docs.sh _build $tag
+          ./scripts/gh/update-docs.sh _build $tag
 
       - name: '📘 Publish'
         if: ${{ github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v') }}
@@ -59,13 +59,14 @@ jobs:
           keep_files: true
           user_name: 'William King Noel Bot'
           user_email: 'adrestia@iohk.io'
+          commit_message: 'docs: ${{ github.event.head_commit.message }}'
 
   bump_sh:
     if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-20.04
     steps:
       - name: '📥 Checkout repository'
-        uses: actions/checkout@v1
+        uses: actions/checkout@v2
       - name: 'Set up Ruby'
         uses: ruby/setup-ruby@v1
         with:
@@ -75,7 +76,7 @@ jobs:
       - name: 'Install dependencies'
         run: 'sudo snap install yq'
       - name: 'Update Release in Bump.sh'
-        run: './scripts/update-bump.sh'
+        run: './scripts/gh/update-bump.sh'
         env:
           BUMP_SH_DOC_ID: ${{ secrets.BUMP_SH_DOC_ID }}
           BUMP_SH_TOKEN: ${{ secrets.BUMP_SH_TOKEN }}

.github/workflows/windows.yml

@@ -112,7 +112,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
         with:
-          fetch-depth: 0
+          fetch-depth: 1
       - name: "Advance windows-tests-pass and all-tests-pass branches"
         if: github.ref == 'ref/heads/master'
         shell: bash

.gitignore

@@ -18,7 +18,7 @@ cabal.sandbox.config
 *.hp
 *.eventlog
 .stack-work/
-*.project.local
+*.project.local*
 .HTF/
 .ghc.environment.*
 .ghci
@@ -38,3 +38,6 @@ result*
 
 ### Release scripts output
 /GENERATED_*.md
+
+### Docs build
+/_build/

nix/haskell.nix

@@ -227,6 +227,9 @@ let
         # Avoid this error on the windows build:
         #   Wrap.hsc:96:10: fatal error: regex.h: No such file or directory
         packages.regex-posix.flags._regex-posix-clib = stdenv.hostPlatform.isWindows;
+
+        # Lets us put the pretty-simple tool in shell.nix.
+        packages.pretty-simple.flags.buildexe = true;
       }
 
       # Enable profiling on executables if the profiling argument is set.

nix/release-package.nix

@@ -56,8 +56,10 @@ pkgs.stdenv.mkDerivation {
   exeName = lib.getName exe.name;
   buildPhase = ''
     mkdir -p $out/nix-support $name
-    cp -nR ${lib.concatMapStringsSep " " (exe: "${exe}/bin/*") exes} $name
-    chmod -R 755 $name
+    cp --no-preserve=timestamps --no-clobber --recursive \
+      ${lib.concatMapStringsSep " " (exe: "${exe}/bin/*") exes} \
+      $name
+    chmod -R +w $name
 
   '' + lib.optionalString isMacOS ''
     # Rewrite library paths to standard non-nix locations
@@ -79,18 +81,18 @@ pkgs.stdenv.mkDerivation {
 
     # make a separate configuration package if needed
     if [ -d ${exe}/configuration ]; then
-      cp --no-preserve=mode -R ${exe}/configuration .
+      cp --no-preserve=mode,timestamps -R ${exe}/configuration .
 
-      ( cd configuration; zip -r $out/${exe.name}-configuration.zip . )
-      echo "file binary-dist $out/${exe.name}-configuration.zip" >> $out/nix-support/hydra-build-products
+      ( cd configuration; zip -r $out/$name-configuration.zip . )
+      echo "file binary-dist $out/$name-configuration.zip" >> $out/nix-support/hydra-build-products
     fi
 
     # make a separate deployments configuration package if needed
     if [ -d ${exe}/deployments ]; then
-      cp --no-preserve=mode -R ${exe}/deployments .
+      cp --no-preserve=mode,timestamps -R ${exe}/deployments .
 
-      ( cd deployments; zip -r $out/${exe.name}-deployments.zip . )
-      echo "file binary-dist $out/${exe.name}-deployments.zip" >> $out/nix-support/hydra-build-products
+      ( cd deployments; zip -r $out/$name-deployments.zip . )
+      echo "file binary-dist $out/$name-deployments.zip" >> $out/nix-support/hydra-build-products
     fi
   '';