Generating an HTML report for oheap encoding.

auto.sh: Work automating the whole per-host measurement process.
oilshell · Dec 8, 2017 · 8ec5f4f6a3a4d26fa64110fe985991c441b1e079 · 8ec5f4f
1 parent 58213dd
commit 8ec5f4f6a3a4d26fa64110fe985991c441b1e079
Unified Split

Showing with 114 additions and 51 deletions.

+66 −45 benchmarks/auto.sh

+45 −5 benchmarks/oheap.sh

+2 −0 benchmarks/report.R

+1 −1 benchmarks/report.sh
diff --git a/benchmarks/auto.sh b/benchmarks/auto.sh
@@ -27,44 +27,6 @@ _banner() {
  echo -----
 }

-# Run the whole benchmark from a clean git checkout.
-#
-# Similar to scripts/release.sh build-and-test.
-all() {
-  test/spec.sh install-shells
-
-  # Technically we need build-essential too?
-  sudo apt install python-dev
-
-  build/dev.sh all
-  build/codegen.sh lexer
-
-  _banner 'OSH dev build'
-  bin/osh -c 'echo OSH dev build'
-
-  build/prepare.sh configure
-  build/prepare.sh build-python
-
-  make _bin/oil.ovm
-  # This does what 'install' does.
-  scripts/run.sh make-bin-links
-
-  _banner 'OSH production build'
-
-  _bin/osh -c 'echo OSH production build'
-
-  # Make observations.
-  # TODO: Factor shell-id / host-id here.  Every benchmark will use that.
-
-  # Just write a task file, like _tmp/benchmark-tasks.txt?
-  # And then have a function to execute the tasks.
-  # It has to make the write CSV files?
-
-  benchmarks/osh-parser.sh run
-
-  # Now osh-parser.sh report is run on a single machine.
-}
-
 # Writes a table of host and shells to stdout.  Writes text files and
 # calculates IDs for them as a side effect.
 #
@@ -107,23 +69,82 @@ record-provenance() {
  done
 }

-write-provenance-txt() {
-  local job_id
-  job_id="$(date +%Y-%m-%d__%H-%M-%S)"
+gen-prefix() {
+  local job_id=$1

  local host
  host=$(hostname)

-  # NOTE: This could be a TSV file?
+  echo _tmp/${host}.${job_id}.provenance.txt
+}

-  local out=_tmp/${host}.${job_id}.provenance.txt
+write-provenance-txt() {
+  local job_id
+  job_id="$(date +%Y-%m-%d__%H-%M-%S)"
+
+  local out=${1:-$(gen-prefix $job_id)}

-  # Job ID should be here
  record-provenance $job_id > $out

-  echo "Wrote $out"
+  log "Wrote $out"
+}
+
+measure-all() {
+  local provenance=$1
+  local base_dir=${2:-../benchmark-data}
+
+  benchmarks/vm-baseline.sh measure $provenence $base_dir/vm-baseline
+  benchmarks/osh-runtime.sh measure $provenence $base_dir/osh-runtime
+  benchmarks/osh-parser.sh measure $provenence $base_dir/osh-parser
+}
+
+# Run the whole benchmark from a clean git checkout.
+#
+# Similar to scripts/release.sh build-and-test.
+all() {
+  test/spec.sh install-shells
+
+  # Technically we need build-essential too?
+  sudo apt install python-dev
+
+  build/dev.sh all
+  build/codegen.sh lexer
+
+  _banner 'OSH dev build'
+  bin/osh -c 'echo OSH dev build'
+
+  build/prepare.sh configure
+  build/prepare.sh build-python
+
+  make _bin/oil.ovm
+  # This does what 'install' does.
+  scripts/run.sh make-bin-links
+
+  _banner 'OSH production build'
+
+  _bin/osh -c 'echo OSH production build'
+
+  # Make observations.
+  # TODO: Factor shell-id / host-id here.  Every benchmark will use that.
+
+  # Just write a task file, like _tmp/benchmark-tasks.txt?
+  # And then have a function to execute the tasks.
+  # It has to make the write CSV files?
+
+  local job_id
+  job_id="$(date +%Y-%m-%d__%H-%M-%S)"
+
+  local provenance=$(gen-prefix $job_id)
+
+  record-provenance $job_id > $provenance
+
+  measure-all $provenance
 }

+#
+# Other
+#
+
 demo-tasks() {
  local provenance=$1

diff --git a/benchmarks/oheap.sh b/benchmarks/oheap.sh
@@ -83,14 +83,54 @@ print-csv() {
  print-size oheap xz $BASE_DIR/oheap-compressed/*.xz 
 }

-report() {
-  local sizes=$BASE_DIR/sizes.csv
-  local out_dir=$BASE_DIR/stage1
+# This can be done on any host.
+measure() {
+  encode-all
+  compress-oheap
+  compress-text
+}

+stage1() {
+  local out_dir=$BASE_DIR/stage1
  mkdir -p $out_dir
+  print-csv > $out_dir/sizes.csv
+}

-  print-csv > $sizes
-  benchmarks/report.R oheap $BASE_DIR $out_dir
+print-report() {
+  local in_dir=$1
+  local base_url='../../web/table'
+
+  cat <<EOF
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>OHeap Encoding</title>
+    <script type="text/javascript" src="$base_url/table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="$base_url/table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="benchmarks.css" />
+
+  </head>
+  <body>
+    <p id="home-link">
+      <a href="/">oilshell.org</a>
+    </p>
+    <h2>OHeap Encoding</h2>
+
+    <h3>Encoding Size (KB)</h3>
+
+    <p>Sizes are in KB (powers of 10), not KiB (powers of 2).</p>
+EOF
+  csv2html $in_dir/encoding_size.csv
+
+  cat <<EOF
+    <h3>Encoding Ratios</h3>
+EOF
+  csv2html $in_dir/encoding_ratios.csv
+
+  cat <<EOF
+  </body>
+</html>
+EOF
 }


diff --git a/benchmarks/report.R b/benchmarks/report.R
@@ -327,6 +327,8 @@ OheapReport = function(in_dir, out_dir) {
  # - oheap is 2-7x bigger uncompressed, and 4-12x bigger compressed.
  # - oheap is less compressible than text!

+  # TODO: The ratio needs 2 digits of precision.
+
  sizes %>%
    transmute(oheap_to_text = oheap_none / text_none,
              xz_text = text_xz / text_none,
diff --git a/benchmarks/report.sh b/benchmarks/report.sh
@@ -70,7 +70,7 @@ vm-baseline() {
 oheap() {
  local base_dir=_tmp/oheap

-  benchmarks/oheap.sh report
+  benchmarks/oheap.sh stage1
  stage2 $base_dir
  stage3 $base_dir
 }