Big refactoring of benchmarks.

I'm adding more benchmarks, and I want them to be consistent and share
the same provenance information.

- Port vm-baseline benchmark to the provenance scheme.  Saves some code.
- Rename runtime.sh to osh-runtime.sh
- Use the xargs -n $NUM_COLUMNS pattern in osh-parser.  From the
  provenance, We write the tasks to a file.  Then we execute the tasks.

Testing: The full osh-parser.sh report works.  I added a
'fake-other-host' function so we can test on a single machine.

Also:

- Minor edits to test/wild2.sh after re-running.

These were the scripts we used to test the OSH 0.0 release, e.g.
Python's configure script.

README.md

@@ -93,12 +93,16 @@ Directory Structure
    _build/           # Temporary build files
    _devbuild/        # Developer build files not deleted upon 'make clean'
    _deps/            # build dependencies like re2c
    _tmp/             # Temporary test files and the like
    _tmp/             # Test suites and other temp files
      spec/
      wild/
        raw/
        www/
      osh-parser/
      osh-runtime/
      vm-baseline/
      oheap/
      startup/
      ...
    _release/         # Source release tarballs are put here
      VERSION/        # Published at oilshell.org/release/$VERSION/

benchmarks/auto.sh

@@ -0,0 +1,144 @@
#!/bin/bash
#
# Run all the benchmarks on a given machine.
#
# Usage:
#   ./auto.sh <function name>
#
# List of benchmarks:
#
# - osh-parser
# - virtual-memory.sh -- vm-baseline, or mem-baseline
# - osh-runtime (now called runtime.sh, or wild-run)
# - oheap.sh?  For size, it doesn't need to be run on every machine.
# - startup.sh -- needs CSV
#   - this has many different snippets
#   - and it has strace

set -o nounset
set -o pipefail
set -o errexit

source test/common.sh  # die

_banner() {
  echo -----
  echo "$@"
  echo -----
}

# Run the whole benchmark from a clean git checkout.
#
# Similar to scripts/release.sh build-and-test.
all() {
  test/spec.sh install-shells

  # Technically we need build-essential too?
  sudo apt install python-dev

  build/dev.sh all
  build/codegen.sh lexer

  _banner 'OSH dev build'
  bin/osh -c 'echo OSH dev build'

  build/prepare.sh configure
  build/prepare.sh build-python

  make _bin/oil.ovm
  # This does what 'install' does.
  scripts/run.sh make-bin-links

  _banner 'OSH production build'

  _bin/osh -c 'echo OSH production build'

  # Make observations.
  # TODO: Factor shell-id / host-id here.  Every benchmark will use that.

  # Just write a task file, like _tmp/benchmark-tasks.txt?
  # And then have a function to execute the tasks.
  # It has to make the write CSV files?

  benchmarks/osh-parser.sh run

  # Now osh-parser.sh report is run on a single machine.
}

# Writes a table of host and shells to stdout.  Writes text files and
# calculates IDs for them as a side effect.
#
# The table can be passed to other benchmarks to ensure that their provenance
# is recorded.
#
# TODO: Move to id.sh/provenance.sh?

record-provenance() {
  local job_id=$1

  local host
  host=$(hostname)

  # Write Header of the CSV file that is appended to.
  #echo 'host_name,host_hash,shell_name,shell_hash'

  local tmp_dir=_tmp/host-id/$host
  benchmarks/id.sh dump-host-id $tmp_dir

  local host_hash
  host_hash=$(benchmarks/id.sh publish-host-id $tmp_dir)
  #echo $host $host_hash

  local shell_hash

  #for sh_path in bash dash mksh zsh; do
  for sh_path in bash dash mksh zsh bin/osh _bin/osh; do
    # There will be two different OSH
    local name=$(basename $sh_path)

    tmp_dir=_tmp/shell-id/$name
    benchmarks/id.sh dump-shell-id $sh_path $tmp_dir

    shell_hash=$(benchmarks/id.sh publish-shell-id $tmp_dir)

    #echo "$sh_path ID: $shell_hash"

    echo "$job_id $host $host_hash $sh_path $shell_hash"
  done
}

write-provenance-txt() {
  local job_id
  job_id="$(date +%Y-%m-%d__%H-%M-%S)"

  local host
  host=$(hostname)

  # NOTE: This could be a TSV file?

  local out=_tmp/${host}.${job_id}.provenance.txt

  # Job ID should be here
  record-provenance $job_id > $out

  echo "Wrote $out"
}

demo-tasks() {
  local provenance=$1

  # Strip everything after the first dot.
  local name=$(basename $provenance)
  local job_id=${name%.provenance.txt}

  echo "JOB ID: $job_id"

  # This is the pattern for iterating over shells.
  cat $provenance | while read _ _ _ sh_path _; do
    for i in 1 2 3; do
      echo $i $sh_path
    done
  done
}

"$@"

benchmarks/osh-parser.R

@@ -104,7 +104,9 @@ main = function(argv) {
    select(-c(host_name, host_hash, shell_name, shell_hash)) ->
    all_times

  Log('summary(all_times):')
  print(summary(all_times))
  Log('head(all_times):')
  print(head(all_times))

  # Summarize rates by platform/shell
@@ -114,6 +116,7 @@ main = function(argv) {
    mutate(lines_per_ms = total_lines / total_ms) ->
    shell_summary

  Log('shell_summary:')
  print(shell_summary)

  # Elapsed seconds for each shell by platform and file