Merge pull request #3578 from input-output-hk/membench-report

bench | process:  Hydra chart rendering

CODEOWNERS

@@ -20,7 +20,7 @@ doc                                                           @docs-access
 README.*                                                      @docs-access
 
 bench/tx-generator                                            @deepfire @MarcFontaine
-bench                                                         @deepfire @denisshevchenko @jutaro @MarcFontaine
+bench                                                         @deepfire @denisshevchenko @jutaro @MarcFontaine @cleverca22
 cardano-tracer                                                @deepfire @denisshevchenko
 nix/workbench                                                 @deepfire @denisshevchenko @jutaro @MarcFontaine
 nix/supervisord-cluster                                       @deepfire @denisshevchenko @jutaro @MarcFontaine

bench/process/membenches_v1.jq

@@ -11,29 +11,37 @@ def format_specs:
      # , post: not
      }]
 , result_specs:
-    [{ key: "RssAvg",  header: "Avg RSS, MB"
-     , path: ["RSS",  "avg"], round: true
+    [{ key: "Average-RSS",      header: "Average RSS",      unit: "MB"
+     , path: ["RSS",  "avg"]
+     , round: true
      }
-    ,{ key: "RssMax",  header: "Max RSS, MB"
-     , path: ["RSS",  "max"], round: true
+    ,{ key: "Peak-RSS",         header: "Peak RSS",         unit: "MB"
+     , path: ["RSS",  "max"]
+     , round: true
      }
-    ,{ key: "HeapAvg", header: "Avg heap, MB"
-     , path: ["Heap", "avg"], round: true
+    ,{ key: "Average-RTS-heap", header: "Average RTS heap", unit: "MB"
+     , path: ["Heap", "avg"]
+     , round: true
      }
-    ,{ key: "HeapMax", header: "Max heap, MB"
-     , path: ["Heap", "max"], round: true
+    ,{ key: "Peak-RTS-heap",    header: "Peak RTS heap",    unit: "MB"
+     , path: ["Heap", "max"]
+     , round: true
      }
-    ,{ key: "WallSec", header: "Wall, s"
-     , path: ["totaltime"], round: true
+    ,{ key: "Wall-clock-time",  header: "Wall clock time",  unit: "s"
+     , path: ["totaltime"]
+     , round: true
      }
-    ,{ key: "CpuMax",  header: "OS CPU, s"
-     , path: ["CentiCpuMax"], scale: 100, round: true
+    ,{ key: "OS-CPU-time",      header: "OS CPU time",      unit: "s"
+     , path: ["CentiCpuMax"]
+     , round: true, scale: 100
      }
-    ,{ key: "MutMax",  header: "Mutator, s"
-     , path: ["CentiMutMax"], scale: 100, round: true
+    ,{ key: "RTS-mutator-time", header: "RTS mutator time", unit: "s"
+     , path: ["CentiMutMax"]
+     , round: true, scale: 100
      }
-    ,{ key: "GCSec",   header: "GC time, s"
-     , path: ["SecGC"], round: true
+    ,{ key: "RTS-GC-time",      header: "RTS GC time",      unit: "s"
+     , path: ["SecGC"]
+     , round: true
      }
     ]
 };

bench/process/process.sh

@@ -32,11 +32,15 @@ main() {
         collect )    op_collect "$@";;
         process )    op_process;;
         render )     op_render;;
+        render-html )
+                     op_render_html;;
+        render-hydra-charts )
+                     op_render_hydra_charts "$@";;
 
         report )     op_collect "$@" | op_process | op_render;;
 
         call )       eval "$@";;
-        * ) echo "ERROR:  operation must be one of:  collect process render report" >&2; exit 1;; esac
+        * ) echo "ERROR:  operation must be one of:  collect process render render-hydra-charts report" >&2; exit 1;; esac
 }
 
 hardcoded_branch='membench'
@@ -115,6 +119,25 @@ function op_render() {
        ' --raw-output
 }
 
+function op_render_html() {
+    jq 'include "render";
+
+        render_html
+       ' --raw-output
+}
+
+function op_render_hydra_charts() {
+    local config='baseline'
+    while test $# -ge 1
+    do case "$1" in
+        --config )          config=$2; shift;;
+        * )                 break;; esac; shift; done
+    jq 'include "render";
+
+        render_hydra_charts_for_config ("'$config'")
+       ' --raw-output
+}
+
 ###
 ### Main
 ###

bench/process/render.jq

@@ -17,6 +17,120 @@ def var(fname; f):
     .[fname] as $val
   | "\($val.mean | f) | \("\($val.relstddev)" | .[:4])";
 
+def render_value(spec; val):
+  (val.mean / (spec.scale // 1))
+  | if spec.round
+    then ceil
+    else . end
+  | tostring;
+
+def render_value_at_index(spec; val; ix):
+  (val.raw[ix] / (spec.scale // 1))
+  | if spec.round
+    then ceil
+    else . end
+  | tostring;
+
+def render_value_relstddev(spec; val):
+  (val.relstddev / (spec.scale // 1) * 100)
+  | tostring
+  | .[:5];
+
+def render_config_hydra_charts (res):
+    . as $x
+  | (res
+      | map(. as $spec
+             | $x[$spec.key] as $val
+             # For each value, produce two Hydra metrics:
+             #  - (id,     value, unit)
+             #  - (id-CoV, CoV,   %)
+             | [ $spec.key
+               , render_value($spec; $val)
+               , $spec.unit
+               , $spec.key + "-coeff-variation"
+               , render_value_relstddev($spec; $val)
+               , "%"
+               ]
+            | join(" ")))
+    as $properties
+
+  | ($properties | join("\n"));
+
+def render_html_config_runs(cf; res; count):
+  [ "<table border=1>\n"
+  , "<caption>Per-run raw data for config '\(.config)'</caption>"
+  , "<tr><th>"
+  , ((cf + res)
+      | map(if .unit != null
+            then [ .header + ", " + .unit ]
+            else [ .header ] end)
+      | add
+      | join("</th><th>"))
+  , "</th></tr>\n"
+  , . as $x
+    |
+    ( [range(count)]
+    | map( . as $ix
+         | cf + res
+         | map(. as $spec
+                | $x[$spec.key] as $val
+                | if .unit != null
+                  then render_value_at_index($spec; $val; $ix)
+                  else $val | tostring end
+                | "<td>" + . + "</td>")
+           | add
+           | ("<tr>" + . + "</tr>\n"))
+    | add)
+  , "\n</table>"
+  ]
+  | add;
+
+def render_html_summary(cf; res):
+  [ "<table border=1>\n"
+  , "<caption>Summary stats for run</caption>"
+  , "<tr><th>"
+  , ((cf + res)
+      | map(if .unit != null
+            then [ .header + ", " + .unit
+                 , .header + ", " + "σ/μ"
+                 ]
+            else [ .header ] end)
+      | add
+      | join("</th><th>"))
+  , "</th></tr>\n"
+  , (.configs
+    | map( . as $x
+         | cf + res
+         | map(. as $spec
+                | $x[$spec.key] as $val
+                | if .unit != null
+                  then [ render_value($spec; $val)
+                       , "</td><td>"
+                       , render_value_relstddev($spec; $val) ]
+                  else [ $val | tostring ] end
+                | ["<td>"] + . + ["</td>"]
+                | add)
+           | add
+           | ("<tr>" + . + "</tr>\n"))
+    | add)
+  , "\n</table>"
+  ]
+  | add;
+
+def render_html:
+    .config_specs as $cf
+  | .result_specs as $res
+  | .runs         as $runs
+
+  | [render_html_summary($cf; $res)]
+    + (.configs
+        | map( . as $config
+               | render_html_config_runs($cf; $res;
+                                         $runs
+                                          | map(select(.config_name == $config.config))
+                                          | length)))
+  | join("\n<hr>\n");
+
 def render_config (format; cf; res):
     . as $x
   | (if format != "csv" then [null] else [] end
@@ -27,9 +141,8 @@ def render_config (format; cf; res):
      (res
       | map(. as $spec
             | $x[$spec.key] as $val
-            | [(($val.mean / ($val.scale // 1))
-               | if $spec.round then ceil else . end)
-              , ($val.relstddev | tostring | .[:5])])
+            | [ render_value($spec; $val)
+              , render_value_relstddev($spec; $val)])
       | add))
     as $columns
 
@@ -88,3 +201,14 @@ def render(header_footer):
     else . end
 | join("\n");
 
+def render_hydra_charts_for_config(config_name):
+  .result_specs as $result_specs
+| .configs
+| map(select(.config == config_name))
+| if . == null or . == []
+  then error ("render_hydra_charts_for_config: unknown config \(config_name)")
+  else .[0]
+  end
+| render_config_hydra_charts ($result_specs)
+# | join("\n")
+;

flake.nix

@@ -143,7 +143,7 @@
     ) // {
       overlay = import ./overlay.nix self;
       hydraJobs.x86_64-linux = {
-        membenches = membench.outputs.packages.x86_64-linux.batch-hydra-report;
+        membenches = membench.outputs.packages.x86_64-linux.batch-report;
         snapshot = membench.outputs.packages.x86_64-linux.snapshot;
       };
       nixosModules = {