Make a report for the osh-runtime benchmark.

Add 'abuild -h' to the suite. Factored code out of the osh-parser benchmark, including CSS, common shell functions, and R code.
oilshell · Dec 5, 2017 · 562ce71616d91ad665737e80df42c5f3baffea94 · 562ce71
1 parent a8bb5c6
commit 562ce71616d91ad665737e80df42c5f3baffea94
Unified Split

Showing with 268 additions and 90 deletions.

+35 −0 benchmarks/benchmarks.css

+12 −0 benchmarks/common.sh

+2 −38 benchmarks/osh-parser.sh

+151 −28 benchmarks/osh-runtime.sh

+64 −20 benchmarks/report.R

+3 −3 scripts/release.sh

+1 −1 test/wild2.sh
diff --git a/benchmarks/benchmarks.css b/benchmarks/benchmarks.css
@@ -0,0 +1,35 @@
+body {
+  margin: 0 auto;
+  width: 60em;
+}
+code {
+  color: green;
+}
+table {
+  margin-left: 3em;
+  font-family: sans-serif;
+}
+td {
+  padding: 8px;  /* override default of 5px */
+}
+h3, h4 {
+  color: darkgreen;
+}
+
+/* these two tables are side by side */
+#shells, #hosts, #raw_times {
+  display: inline-block;
+  vertical-align: top;
+}
+#home-link {
+  text-align: right;
+}
+
+/* columns */
+#osh-ovm, #osh-cpython {
+  background-color: oldlace;
+}
+/* rows */
+.osh-row {
+  background-color: oldlace;
+}
diff --git a/benchmarks/common.sh b/benchmarks/common.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# Common functions for benchmarks.
+#
+
+csv-concat() {
+  tools/csv_concat.py "$@"
+}
+
+csv2html() {
+  web/table/csv2html.py "$@"
+}
diff --git a/benchmarks/osh-parser.sh b/benchmarks/osh-parser.sh
@@ -220,45 +220,8 @@ _print-report() {
    <title>OSH Parser Performance</title>
    <script type="text/javascript" src="$base_url/table-sort.js"></script>
    <link rel="stylesheet" type="text/css" href="$base_url/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="benchmarks.css" />

-    <style>
-      body {
-        margin: 0 auto;
-        width: 60em;
-      }
-      code {
-        color: green;
-      }
-      table {
-        margin-left: 3em;
-        font-family: sans-serif;
-      }
-      td {
-        padding: 8px;  /* override default of 5px */
-      }
-      h3, h4 {
-        color: darkgreen;
-      }
-
-      /* these two tables are side by side */
-      #shells, #hosts, #raw_times {
-        display: inline-block;
-        vertical-align: top;
-      }
-      #home-link {
-        text-align: right;
-      }
-
-      /* columns */
-      #osh-ovm, #osh-cpython {
-        background-color: oldlace;
-      }
-      /* rows */
-      .osh-row {
-        background-color: oldlace;
-      }
-
-    </style>
  </head>
  <body>
    <p id="home-link">
@@ -315,6 +278,7 @@ stage3() {
  local out=$BASE_DIR/index.html
  mkdir -p $(dirname $out)
  _print-report $BASE_DIR/stage2 > $out
+  cp -v benchmarks/benchmarks.css $BASE_DIR
  echo "Wrote $out"
 }

diff --git a/benchmarks/osh-runtime.sh b/benchmarks/osh-runtime.sh
@@ -59,28 +59,54 @@ extract() {
 # Computation
 #

+readonly PY27_DIR=$PWD/Python-2.7.13
+
+cpython-configure() {
+  local sh_path=${1:-bash}
+  local out_dir=${2:-$BASE_DIR/${sh_path}-cpython-configure}
+  mkdir -p $out_dir
+
+  pushd $out_dir
+  time $sh_path $PY27_DIR/configure || true
+  popd
+
+  tree $out_dir
+}
+
+# 18.9 seconds vs 11 seconds above.
+osh-cpython-configure() {
+  cpython-configure $PWD/_bin/osh $BASE_DIR/osh-cpython-configure
+}
+
 conf-task() {
  local raw_dir=$1  # output
  local job_id=$2
  local host=$3
  local host_hash=$4
  local sh_path=$5
  local shell_hash=$6
-  local conf_dir=$7
-
-  echo
-  echo "--- $sh_path $conf_dir ---"
-  echo
+  local task_type=$7
+  local task_arg=$8

  local shell_name=$(basename $sh_path)
-  local dir_name=$(basename $conf_dir)

-  local task_label="${shell_name}-${shell_hash}__${dir_name}"
+  # NOTE: For abuild, this isn't a directory name.
+  local x=$(basename $task_arg)
+  local task_label="${shell_name}-${shell_hash}__${x}"

  local times_out="$PWD/$raw_dir/$host.$job_id.times.csv"
  local vm_out_dir="$PWD/$raw_dir/$host.$job_id.virtual-memory"
-  local conf_out_dir="$PWD/$raw_dir/$host.$job_id.files/$task_label"
-  mkdir -p $vm_out_dir $conf_out_dir
+  local files_out_dir="$PWD/$raw_dir/$host.$job_id.files/$task_label"
+  mkdir -p $vm_out_dir $files_out_dir
+
+  local time_tool=$PWD/benchmarks/time.py
+  local -a TIME_PREFIX=(
+    $time_tool \
+    --output $times_out \
+    --field "$host" --field "$host_hash" \
+    --field "$shell_name" --field "$shell_hash" \
+    --field "$task_type" --field "$task_arg"
+  )

  # Can't use array because of set -u bug!!!  Only fixed in bash 4.4.
  extra_args=''
@@ -93,23 +119,37 @@ conf-task() {
    # Should we add a field here to say it has VM stats?
  fi

-  local time_tool=$PWD/benchmarks/time.py
+  echo
+  echo "--- $sh_path $task_type $task_arg ---"
+  echo

-  pushd $conf_dir >/dev/null
-  touch __TIMESTAMP
+  case $task_type in
+    abuild)
+      # NOTE: $task_arg unused.

-  # exit code, time in seconds, host_hash, shell_hash, path.  \0
-  # would have been nice here!
-  $time_tool \
-    --output $times_out \
-    --field "$host" --field "$host_hash" \
-    --field "$shell_name" --field "$shell_hash" \
-    --field "$conf_dir" -- \
-    "$sh_path" $extra_args ./configure >STDOUT.txt || echo FAILED
+      "${TIME_PREFIX[@]}" -- \
+        "$sh_path" $extra_args benchmarks/testdata/abuild -h \
+        > $files_out_dir/STDOUT.txt
+      ;;
+
+    configure)
+      local conf_dir=$task_arg
+
+      pushd $conf_dir >/dev/null
+      touch __TIMESTAMP
+
+      "${TIME_PREFIX[@]}" -- "$sh_path" $extra_args ./configure \
+        > $files_out_dir/STDOUT.txt

-  find . -type f -newer __TIMESTAMP \
-    | xargs -I {} -- cp --verbose {} $conf_out_dir
-  popd >/dev/null
+      find . -type f -newer __TIMESTAMP \
+        | xargs -I {} -- cp --verbose {} $files_out_dir
+      popd >/dev/null
+      ;;
+
+    *)
+      die "Invalid task type $task_type"
+      ;;
+  esac
 }

 # TODO:
@@ -159,13 +199,17 @@ print-tasks() {
        ;;
    esac

+    # NOTE: 'help' is a dummy label.
+    echo $job_id $host_name $host_hash $sh_path $shell_hash abuild help
+
    conf-dirs | xargs -n 1 -- \
-      echo $job_id $host_name $host_hash $sh_path $shell_hash
+      echo $job_id $host_name $host_hash $sh_path $shell_hash configure
+
  done
 }

-readonly HEADER='status,elapsed_secs,host_name,host_hash,shell_name,shell_hash,conf_dir'
-readonly NUM_COLUMNS=6  # 5 from provenence, 1 for dir
+readonly HEADER='status,elapsed_secs,host_name,host_hash,shell_name,shell_hash,task_type,task_arg'
+readonly NUM_COLUMNS=7  # 5 from provenence, then task_type / task_arg

 all() {
  local provenance=$1
@@ -186,8 +230,10 @@ all() {
  print-tasks $provenance > $tasks

  # Run them all
-  #head -n 2 $tasks | xargs -n $NUM_COLUMNS -- $0 conf-task $raw_dir
-  cat $tasks | xargs -n $NUM_COLUMNS -- $0 conf-task $raw_dir
+  #head -n 2 $tasks |
+  time cat $tasks |
+    xargs -n $NUM_COLUMNS -- $0 conf-task $raw_dir ||
+    die "Some tasks failed."

  cp -v $provenance $raw_dir
 }
@@ -213,6 +259,83 @@ stage2() {
  tree $out
 }

+_print-report() {
+  local in_dir=$1
+  local base_url='../../web/table'
+
+  cat <<EOF
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>OSH Runtime Performance</title>
+    <script type="text/javascript" src="$base_url/table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="$base_url/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="benchmarks.css" />
+
+  </head>
+  <body>
+    <p id="home-link">
+      <a href="/">oilshell.org</a>
+    </p>
+    <h2>OSH Runtime Performance</h2>
+
+    <h3>Elapsed Time by Shell (milliseconds)</h3>
+
+    <p>Some benchmarks call many external tools, while some exercise the shell
+    interpreter itself.  Parse time is included.</p>
+EOF
+  csv2html $in_dir/times.csv
+
+  cat <<EOF
+    <h3>Memory Used to Run</h3>
+
+    <p>For <code>osh-ovm</code>.</p>
+EOF
+  #web/table/csv2html.py $in_dir/virtual-memory.csv
+
+  cat <<EOF
+
+    <h3>Shell and Host Details</h3>
+EOF
+  csv2html $in_dir/shells.csv
+  csv2html $in_dir/hosts.csv
+
+cat <<EOF
+    <h3>Raw Data</h3>
+EOF
+  #web/table/csv2html.py $in_dir/raw-data.csv
+
+cat <<EOF
+    <h3>Parse Time Breakdown by File</h3>
+
+    <h4>Elasped Time in milliseconds</h4>
+EOF
+  #web/table/csv2html.py $in_dir/elapsed.csv
+  cat <<EOF
+
+    <h4>Parsing Rate in lines/millisecond</h4>
+EOF
+  #web/table/csv2html.py $in_dir/rate.csv
+  cat <<EOF
+  </body>
+</html>
+EOF
+}
+
+stage3() {
+  local out=$BASE_DIR/index.html
+  mkdir -p $(dirname $out)
+  _print-report $BASE_DIR/stage2 > $out
+  cp -v benchmarks/benchmarks.css $BASE_DIR
+  echo "Wrote $out"
+}
+
+report() {
+  stage1
+  stage2
+  stage3
+}
+
 #
 # Non-configure scripts
 #