Publish all test results under oilshell.org/release/$VERSION/.

- Tweak the unit test runner, make a report with R, and hook it up to
  scripts/release.sh.

benchmarks/report.R

@@ -13,11 +13,6 @@ source('benchmarks/common.R')

options(stringsAsFactors = F)

Log = function(fmt, ...) {
  cat(sprintf(fmt, ...))
  cat('\n')
}

sourceUrl = function(path) {
  sprintf('https://github.com/oilshell/oil/blob/master/%s', path)
}

doc/release-index.md

@@ -22,15 +22,19 @@ Oil Version 0.3.alpha1

- [Github Wiki for oilshell/oil](https://github.com/oilshell/oil/wiki)

### Tests and Metrics
### Tests Results

- [Spec Test Results](test/spec.wwz/).  Test OSH behavior against
  that of existing shells.
- [Wild Test Results](test/wild.wwz/).  Parsing and translating thousands of
  shell scripts with OSH.
- [Line Counts](metrics/line-counts.wwz/).  How much code is in Oil?
- [Spec Tests](test/spec.wwz/).  Test OSH behavior against that of existing
  shells.
- [Wild Tests](test/wild.wwz/).  Parsing and translating thousands of shell
  scripts with OSH.
- [Unit Tests](test/unit.wwz/).  Python unit test results.
- [Gold Tests](test/gold.wwz/log.txt).  Comparisons against bash (using
  implicit assertions, no golden output.)
- [osh2oil Tests](test/osh2oil.wwz/log.txt).  Test the conversion of OSH to
  Oil.

### Benchmarks
### Benchmarks and Metrics

- [OSH Parser Performance](benchmarks.wwz/osh-parser/).  How fast does OSH
  parse compared to other shells?
@@ -39,6 +43,7 @@ Oil Version 0.3.alpha1
- [Virtual Memory Baseline](benchmarks.wwz/vm-baseline/).  How much memory to
  shells use at startup?
- [OHeap](benchmarks.wwz/oheap/).  Metrics for a possible AST encoding format.
- [Line Counts](metrics/line-counts.wwz/).  How much code is in Oil?


<!-- TODO: 

scripts/release.sh

@@ -95,8 +95,9 @@ build-and-test() {

  build/dev.sh all  # for {libc,fastlex}.so, needed to crawl deps

  test/unit.sh all
  test/unit.sh run-for-release
  test/osh2oil.sh run-for-release
  test/gold.sh run-for-release

  build/prepare.sh configure
  build/prepare.sh build-python
@@ -333,12 +334,28 @@ _link() {
  ln -s -f -v --no-target-directory "$@"
}

compress-txt() {
  local name=$1

  log "--- test/$name"
  local out="$root/test/$name.wwz"
  pushd _tmp/$name
  time zip -r -q $out .  # recursive, quiet
  popd
}

compress() {
  local root=$PWD/_release/VERSION/

  # TODO:
  #log "--- test/unit"
  #log "--- test/osh2oil"
  # There is a single log.txt file in _tmp/{osh2oil,gold}
  compress-txt osh2oil
  compress-txt gold

  log "--- test/unit"
  local out="$root/test/unit.wwz"
  pushd _tmp/unit
  time zip -r -q $out .  # recursive, quiet
  popd

  log "--- test/spec"
  local out="$root/test/spec.wwz"

test/gold.sh

@@ -25,15 +25,14 @@ _compare() {

  if ! diff -u _tmp/left.txt _tmp/right.txt; then
    echo FAIL
    return 1
    exit 1
  fi

  if test $left_status != $right_status; then
    echo "FAIL: Got status $right_status but expected $left_status"
    return 1
    exit 1
  fi

  echo PASS
  return 0
}

@@ -115,7 +114,8 @@ declare() { _compare gold/declare.sh demo; }
# Needs declare -p
scope() { _compare gold/scope.sh; }

all() {

readonly -a PASSING=(
  # FLAKY: This one differs by timestamp
  version-text

@@ -136,6 +136,26 @@ all() {
  # There are slight differences in the number of syscalls reported.  Not sure
  # of the cause.
  #startup-benchmark
)

all-passing() {
  for t in "${PASSING[@]}"; do
    # fail calls 'exit 1'
    $t
    echo "OK  $t"
  done

  echo
  echo "All gold tests passed."
}

run-for-release() {
  local out_dir=_tmp/gold
  mkdir -p $out_dir

  all-passing | tee $out_dir/log.txt

  echo "Wrote $out_dir/log.txt"
}

"$@"

test/osh2oil.sh

@@ -923,7 +923,7 @@ readonly -a PASSING=(

all-passing() {
  for t in "${PASSING[@]}"; do
    # fail calls 'exit 1' i this fails.
    # fail calls 'exit 1'
    $t
    echo "OK  $t"
  done

test/report.R

@@ -0,0 +1,55 @@
#!/usr/bin/Rscript
#
# report.R

library(dplyr)
library(ggplot2)

options(stringsAsFactors = F)

source('benchmarks/common.R')

UnitTestReport = function(in_dir, out_dir) {
  tasks = read.csv(file.path(in_dir, 'tasks.csv'))

  tasks %>% filter(status != 0) -> failed
  if (nrow(failed) != 0) {
    print(failed)
    stop('Some tasks failed')
  }

  tasks %>% 
    mutate(elapsed_ms = elapsed_secs * 1000) %>%
    select(-c(status, elapsed_secs)) %>%
    select(c(elapsed_ms, test, test_HREF)) ->
    tasks

  precision = SamePrecision(0)
  writeCsv(tasks, file.path(out_dir, 'report'), precision)
}

main = function(argv) {
  action = argv[[1]]
  in_dir = argv[[2]]
  out_dir = argv[[3]]

  if (action == 'unit') {
    UnitTestReport(in_dir, out_dir)

  } else if (action == 'spec') {
    # TODO

  } else {
    Log("Invalid action '%s'", action)
    quit(status = 1)
  }
  Log('PID %d done', Sys.getpid())
}


if (length(sys.frames()) == 0) {
  # increase ggplot font size globally
  #theme_set(theme_grey(base_size = 20))

  main(commandArgs(TRUE))
}

test/unit.sh

@@ -73,10 +73,9 @@ all() {
  echo "All unit tests passed."
}

# TODO: Use benchmarks/time.py to make a table.
# all should just enumerate tasks
#
# tests-to-run | xargs -n 1 $0 check-for-success
# For _release/VERSION
#

run-test-and-log() {
  local tasks_csv=$1
@@ -88,15 +87,15 @@ run-test-and-log() {

  benchmarks/time.py --out $tasks_csv \
    --field $t --field "$t.txt" -- \
    $t > $log
    $t >$log 2>&1
}

run-for-release() {
run-all-and-log() {
  local out_dir=_tmp/unit
  mkdir -p $out_dir
  rm -r -f $out_dir/*

  local tasks_csv=$out_dir/TASKS.csv
  local tasks_csv=$out_dir/tasks.csv

  local status=0

@@ -120,6 +119,7 @@ run-for-release() {
  echo "All unit tests passed."
}


source benchmarks/common.sh

# TODO: It would be nice to have timestamps of the underlying CSV files and
@@ -147,7 +147,7 @@ print-report() {
    <h2>Unit Test Results</h2>

EOF
  csv2html $in_dir/TASKS.csv
  csv2html $in_dir/report.csv

  cat <<EOF
  </body>
@@ -165,8 +165,15 @@ EOF

write-report() {
  local out=_tmp/unit/index.html
  test/report.R unit _tmp/unit _tmp/unit
  print-report > $out
  echo "Wrote $out"
}

# Called by scripts/release.sh.
run-for-release() {
  run-all-and-log
  write-report
}

"$@"