Move benchmarks to cicero

nix/cells/cloud/actions.nix

@@ -41,4 +41,23 @@
       }
     '';
   };
+
+  "plutus/benchmark" = {
+    task = "benchmark";
+    io = ''
+      // This is a CUE expression that defines what events trigger a new run of this action.
+      // There is no documentation for this yet. Ask SRE if you have trouble changing this.
+
+      let github = {
+        #input: "GitHub event"
+        #repo: "input-output-hk/plutus"
+      }
+
+      #lib.merge
+      #ios: [
+        #lib.io.github_push & github,
+        { #lib.io.github_pr, github, #target_default: false },
+      ]
+    '';
+  };
 }

nix/cells/cloud/pipelines.nix

@@ -0,0 +1,85 @@
+{ cell
+, inputs
+}:
+/*
+  This code needs to be separate from any of the cells it uses
+  to prevent infinite recursion. Specifically the `flakeOutputTasks`
+  call triggers the error when called with any attribute that relies
+  on the cell that the pipelines are defined in.
+  That is why this is currently located in a separate cloud cell
+  and not the automation cell.
+*/
+let
+  inherit (inputs.nixpkgs) lib system;
+  inherit (inputs.tullia) flakeOutputTasks taskSequence;
+  inherit (inputs.cells.automation) ciJobs;
+
+  common =
+    { config
+    , ...
+    }: {
+      preset = {
+        # needed on top-level task to set runtime options
+        nix.enable = true;
+
+        github-ci = {
+          # Tullia tasks can run locally or on Cicero.
+          # When no facts are present we know that we are running locally and vice versa.
+          # When running locally, the current directory is already bind-mounted
+          # into the container, so we don't need to fetch the source from GitHub
+          # and we don't want to report a GitHub status.
+          enable = config.actionRun.facts != { };
+          repo = "input-output-hk/plutus";
+          sha = config.preset.github-ci.lib.getRevision "GitHub event" null;
+        };
+      };
+    };
+
+  # Only build the required job, other jobs might change and become unavailable
+  # This is necessary because cicero only evaluates the task list on master,
+  # instead of the current branch
+  ciJobsToBuild = lib.getAttrs [ "required" ] ciJobs;
+
+  ciTasks =
+    (__mapAttrs
+      (_: flakeOutputTask: { ... }: {
+        imports = [ common flakeOutputTask ];
+
+        memory = 1024 * 8;
+        nomad.resources.cpu = 10000;
+      })
+      (flakeOutputTasks [ system "automation" "ciJobs" ] {
+        # Replicate flake output structure here, so that the generated nix build
+        # commands reference the right output relative to the top-level of the flake.
+        outputs.${system}.automation.ciJobs = ciJobsToBuild;
+      }));
+
+  ciTasksSeq = taskSequence "ci/" ciTasks (__attrNames ciTasks);
+
+  benchmark = { config, ... }: {
+    preset = {
+      nix.enable = true;
+    };
+    command.runtimeInputs = [
+    ];
+    command.text = ''
+      echo TESTSTSTSTSSTSSTSTSTS
+      ls -la
+      pwd
+      nix develop --command "bash ${inputs.self + /scripts/ci-plutus-benchmark.sh}"
+    '';
+  };
+
+in
+if system == "x86_64-linux" then
+  ciTasks // # for running separately
+  ciTasksSeq // # for running in an arbitrary sequence
+  {
+    "ci" = { ... }: {
+      imports = [ common ];
+      after = __attrNames ciTasksSeq;
+    };
+
+    "benchmark" = benchmark;
+  }
+else { }

scripts/ci-plutus-benchmark.sh

@@ -25,6 +25,9 @@
 
 set -e
 
+PR_NUMBER=4956
+BENCHMARK_NAME=nofib
+
 if [ -z "$PR_NUMBER" ] ; then
    echo "[ci-plutus-benchmark]: 'PR_NUMBER' is not set! Exiting"
    exit 1
@@ -44,9 +47,10 @@ echo "[ci-plutus-benchmark]: Clearing caches with cabal clean ..."
 cabal clean
 
 echo "[ci-plutus-benchmark]: Running benchmark for PR branch ..."
-nix-shell --run "cabal bench $BENCHMARK_NAME >bench-PR.log 2>&1"
+# nix-shell --run "cabal bench $BENCHMARK_NAME >bench-PR.log 2>&1"
 # TODO(std) check that this works 
 # nix develop --command "cabal bench $BENCHMARK_NAME >bench-PR.log 2>&1"
+cabal bench $BENCHMARK_NAME >bench-PR.log 2>&1
 
 echo "[ci-plutus-benchmark]: fetching origin ..."
 git fetch origin
@@ -60,9 +64,10 @@ echo "[ci-plutus-benchmark]: Clearing caches with cabal clean ..."
 cabal clean
 
 echo "[ci-plutus-benchmark]: Running benchmark for base branch ..."
-nix-shell --run "cabal bench $BENCHMARK_NAME >bench-base.log 2>&1"
+# nix-shell --run "cabal bench $BENCHMARK_NAME >bench-base.log 2>&1"
 # TODO(std) check that this works 
 # nix develop --command "cabal bench $BENCHMARK_NAME >bench-base.log 2>&1"
+cabal bench $BENCHMARK_NAME >bench-base.log 2>&1
 
 git checkout "$PR_BRANCH_REF"  # .. so we use the most recent version of the comparison script
 
@@ -81,9 +86,10 @@ EOF
 echo -e "</details>"
 } > bench-compare-result.log
 
-nix-shell -p jq --run "jq -Rs '.' bench-compare-result.log >bench-compare.json"
+# nix-shell -p jq --run "jq -Rs '.' bench-compare-result.log >bench-compare.json"
 # TODO(std) check that this works
 # nix develop --command "jq -Rs '.' bench-compare-result.log >bench-compare.json"
+jq -Rs '.' bench-compare-result.log >bench-compare.json
 
 echo "[ci-plutus-benchmark]: Posting results to GitHub ..."
 curl -s -H "Authorization: token $(</run/keys/buildkite-github-token)" -X POST -d "{\"body\": $(<bench-compare.json)}" "https://api.github.com/repos/input-output-hk/plutus/issues/${PR_NUMBER}/comments"