Enhance the benchmark reports and make them more consistent.

- Better formatting for virtual memory tables - Change them to use megabytes (powers of 10) - Add analysis for the vm-baseline metric - Share pipeline code in report.sh
oilshell · Dec 7, 2017 · 09318582e08ba3aa489447719892fa76fdffb5df · 0931858
1 parent 1edc52d
commit 09318582e08ba3aa489447719892fa76fdffb5df
Unified Split

Showing with 140 additions and 67 deletions.

+1 −1 benchmarks/benchmarks.css

+3 −25 benchmarks/osh-parser.sh

+4 −25 benchmarks/osh-runtime.sh

+39 −5 benchmarks/report.R

+74 −0 benchmarks/report.sh

+19 −11 benchmarks/virtual-memory.sh
diff --git a/benchmarks/benchmarks.css b/benchmarks/benchmarks.css
@@ -1,6 +1,6 @@
 body {
  margin: 0 auto;
-  width: 60em;
+  width: 70em;
 }
 code {
  color: green;
diff --git a/benchmarks/osh-parser.sh b/benchmarks/osh-parser.sh
@@ -193,22 +193,13 @@ stage1() {
  wc -l $out/*
 }

-stage2() {
-  local out=$BASE_DIR/stage2
-  mkdir -p $out
-
-  benchmarks/report.R osh-parser $BASE_DIR/stage1 $out
-
-  tree $out
-}
-
 # TODO:
 # - maybe rowspan for hosts: flanders/lisa
 #   - does that interfere with sorting?
 #
 # NOTE: not bothering to make it sortable now.  Just using the CSS.

-_print-report() {
+print-report() {
  local in_dir=$1
  local base_url='../../web/table'

@@ -240,7 +231,8 @@ EOF
  cat <<EOF
    <h3>Memory Used to Parse</h3>

-    <p>For <code>osh-ovm</code>.</p>
+    <p>Running under <code>osh-ovm</code>.  Memory usage is measured in MB
+    (powers of 10), not MiB (powers of 2).</p>
 EOF
  web/table/csv2html.py $in_dir/virtual-memory.csv

@@ -273,20 +265,6 @@ EOF
 EOF
 }

-stage3() {
-  local out=$BASE_DIR/index.html
-  mkdir -p $(dirname $out)
-  _print-report $BASE_DIR/stage2 > $out
-  cp -v benchmarks/benchmarks.css $BASE_DIR
-  echo "Wrote $out"
-}
-
-report() {
-  stage1
-  stage2
-  stage3
-}
-
 time-test() {
  benchmarks/time.py \
    --field bash --field foo.txt --output _tmp/bench.csv \
diff --git a/benchmarks/osh-runtime.sh b/benchmarks/osh-runtime.sh
@@ -254,16 +254,7 @@ stage1() {
  #local raw_dir=${1:-../benchmark-data/osh-parser}
 }

-stage2() {
-  local out=$BASE_DIR/stage2
-  mkdir -p $out
-
-  benchmarks/report.R osh-runtime $BASE_DIR/stage1 $out
-
-  tree $out
-}
-
-_print-report() {
+print-report() {
  local in_dir=$1
  local base_url='../../web/table'

@@ -293,7 +284,9 @@ EOF
  cat <<EOF
    <h3>Memory Used to Run</h3>

-    <p>Running under <code>osh-ovm</code>.</p>
+    <p>Running under <code>osh-ovm</code>.  Memory usage is measured in MB
+    (powers of 10), not MiB (powers of 2).</p>
+
 EOF
  web/table/csv2html.py $in_dir/virtual-memory.csv

@@ -310,20 +303,6 @@ EOF
 EOF
 }

-stage3() {
-  local out=$BASE_DIR/index.html
-  mkdir -p $(dirname $out)
-  _print-report $BASE_DIR/stage2 > $out
-  cp -v benchmarks/benchmarks.css $BASE_DIR
-  echo "Wrote $out"
-}
-
-report() {
-  stage1
-  stage2
-  stage3
-}
-
 #
 # Non-configure scripts
 #
diff --git a/benchmarks/report.R b/benchmarks/report.R
@@ -172,11 +172,15 @@ ParserReport = function(in_dir, out_dir) {
    select(-c(shell_name, shell_hash)) %>%
    filter(shell_label == 'osh-ovm') %>%
    select(-c(shell_label)) %>%
-    spread(key = metric_name, value = metric_value) %>%
+    rename(kib = metric_value) %>%
+    mutate(megabytes = kib * 1024 / 1e6) %>%
+    select(-c(kib)) %>%
+    spread(key = metric_name, value = megabytes) %>%
    left_join(lines_by_filename, by = c('filename')) %>%
    arrange(host, num_lines) %>%
    mutate(filename_HREF = sourceUrl2(filename)) %>% 
-    select(c(host, VmPeak, VmRSS, num_lines, filename, filename_HREF)) ->
+    rename(VmPeak_MB = VmPeak, VmRSS_MB = VmRSS) %>%
+    select(c(host, VmRSS_MB, VmPeak_MB, num_lines, filename, filename_HREF)) ->
    vm_table

  Log('\n')
@@ -276,9 +280,15 @@ RuntimeReport = function(in_dir, out_dir) {
  vm %>%
    filter(shell_name == 'osh') %>%
    select(-c(shell_name, shell_hash)) %>%
-    mutate(mem_name = paste(metric_name, event, sep = '_')) %>%
-    select(-c(metric_name, event)) %>%
-    spread(key = c(mem_name), value = metric_value) ->
+    rename(kib = metric_value) %>%
+    mutate(megabytes = kib * 1024 / 1e6) %>%
+    select(-c(kib)) %>%
+    mutate(mem_name = paste(event, metric_name, 'MB', sep = '_')) %>%
+    select(-c(event, metric_name)) %>%
+    spread(key = c(mem_name), value = megabytes) %>%
+    select(c(host, task_arg,
+             parser_VmRSS_MB, parser_VmPeak_MB,
+             runtime_VmRSS_MB, runtime_VmPeak_MB)) ->
    vm

  Log('VM:')
@@ -333,6 +343,27 @@ OheapReport = function(in_dir, out_dir) {
  Log('Wrote %s', out_dir)
 }

+VmBaselineReport = function(in_dir, out_dir) {
+  vm = read.csv(file.path(in_dir, 'vm-baseline.csv'))
+  #print(vm)
+
+  # TODO: Should label osh-ovm and osh-cpython, like above.
+
+  vm %>%
+    rename(kib = metric_value) %>%
+    mutate(megabytes = kib * 1024 / 1e6) %>%
+    select(-c(kib)) %>%
+    spread(key = c(metric_name), value = megabytes) %>%
+    rename(VmPeak_MB = VmPeak, VmRSS_MB = VmRSS) %>%
+    select(c(host, shell_name, shell_hash, VmRSS_MB, VmPeak_MB)) %>%
+    arrange(VmPeak_MB) ->
+    vm
+
+  print(vm)
+
+  writeCsv(vm, file.path(out_dir, 'vm-baseline'))
+}
+
 main = function(argv) {
  action = argv[[1]]
  in_dir = argv[[2]]
@@ -344,6 +375,9 @@ main = function(argv) {
  } else if (action == 'osh-runtime') {
    RuntimeReport(in_dir, out_dir)

+  } else if (action == 'vm-baseline') {
+    VmBaselineReport(in_dir, out_dir)
+
  } else if (action == 'oheap') {
    OheapReport(in_dir, out_dir)

diff --git a/benchmarks/report.sh b/benchmarks/report.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Usage:
+#   ./report.sh <function name>
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+
+# TODO: Move stuff from osh-parser.sh, osh-runtime.sh, etc.
+#
+# stage1 : concatenate files from different machines
+# stage2 : make CSV files with report.R
+# stage3 : make HTML files.  Call the 'print-report function here.
+
+
+stage2() {
+  local base_dir=$1  # _tmp/{osh-parser,osh-runtime,...}
+  local action=$(basename $base_dir)
+
+  local out=$base_dir/stage2
+  mkdir -p $out
+
+  benchmarks/report.R $action $base_dir/stage1 $out
+
+  tree $out
+}
+
+stage3() {
+  local base_dir=$1  # _tmp/{osh-parser,osh-runtime,...}
+  local name=$(basename $base_dir)
+  local script=benchmarks/$name.sh
+
+  local out=$base_dir/index.html
+  mkdir -p $(dirname $out)
+
+  $script print-report $base_dir/stage2 > $out
+
+  cp -v benchmarks/benchmarks.css $base_dir
+  echo "Wrote $out"
+}
+
+osh-parser() {
+  local base_dir=_tmp/osh-parser
+
+  benchmarks/osh-parser.sh stage1
+  stage2 $base_dir
+  stage3 $base_dir
+}
+
+osh-runtime() {
+  local base_dir=_tmp/osh-runtime
+
+  benchmarks/osh-runtime.sh stage1
+  stage2 $base_dir
+  stage3 $base_dir
+}
+
+# NOTE: This is just processing
+vm-baseline() {
+  local base_dir=_tmp/vm-baseine
+
+  benchmarks/virtual-memory.sh stage1
+}
+
+# This is one is specific to a particular machine.
+oheap() {
+  local base_dir=_tmp/oheap
+
+  benchmarks/oheap.sh report
+}
+
+"$@"
diff --git a/benchmarks/virtual-memory.sh b/benchmarks/virtual-memory.sh
@@ -9,21 +9,20 @@ set -o errexit

 source test/common.sh  # log

+readonly BASE_DIR=_tmp/vm-baseline
+
 # TODO: Call this from benchmarks/auto.sh.

 vm-baseline() {
  local provenance=$1
  local base_dir=${2:-_tmp/vm-baseline}
  #local base_dir=${2:-../benchmark-data/vm-baseline}

-  # Strip everything after the first dot.
  local name=$(basename $provenance)
-  local job_id=${name%%.*}
-
-  log "--- Job $job_id ---"
+  local prefix=${name%.provenance.txt}  # strip suffix

  local host=$(hostname)
-  local out_dir="$base_dir/$host.$job_id"
+  local out_dir="$base_dir/$prefix"
  mkdir -p $out_dir

  # Fourth column is the shell.
@@ -40,15 +39,24 @@ vm-baseline() {
  ls -l $out_dir
 }

-csv-demo() {
-  local -a job_dirs=(_tmp/vm-baseline/lisa.2017-*)
-  benchmarks/virtual_memory.py baseline ${job_dirs[-1]}
+# Run a single file through stage 1 and report.
+demo() {
+  local -a job_dirs=($BASE_DIR/lisa.2017-*)
+  local dir1=$BASE_DIR/stage1
+  local dir2=$BASE_DIR/stage2
+
+  mkdir -p $dir1 $dir2
+  
+  benchmarks/virtual_memory.py baseline ${job_dirs[-1]} \
+    > $dir1/vm-baseline.csv
+
+  benchmarks/report.R vm-baseline $dir1 $dir2
 }

 # Combine CSV files.
-baseline-csv() {
-  local raw_dir=$1
-  local out=_tmp/vm-baseline/stage1
+stage1() {
+  local raw_dir=${1:-$BASE_DIR/raw}
+  local out=$BASE_DIR/stage1
  mkdir -p $out

  # Globs are in lexicographical order, which works for our dates.