[test/stateful] Aggregate results in HTML, like test/spec

Run with xargs, process *.task.txt files, make a table, etc.

soil/travis.sh

@@ -234,8 +234,8 @@ make-job-wwz() {
   #        Note that that index references /web/{base,soil}.css, outside the .wwz
   #        osh-summary.html uses table-sort.js and ajax.js
   zip -r $wwz \
-    index.html _tmp/soil _tmp/spec _tmp/syscall _tmp/benchmark-data \
-    _tmp/metrics \
+    index.html _tmp/soil _tmp/spec _tmp/stateful \
+    _tmp/syscall _tmp/benchmark-data _tmp/metrics \
     mycpp/_ninja/*.{html,txt,tsv} mycpp/_ninja/{tasks,gen} \
     web/{base,spec-code,spec-tests,spec-cpp,line-counts}.css web/ajax.js \
     web/table/table-sort.{css,js} \

soil/worker.sh

@@ -109,7 +109,7 @@ lint                test/lint.sh soil-run            -
 typecheck-slice     types/oil-slice.sh soil-run      -
 typecheck-other     types/run.sh soil-run            -
 unit                test/unit.sh soil-run            -
-stateful            test/stateful.sh soil-run     -
+stateful            test/stateful.sh soil-run        _tmp/stateful/index.html
 parse-errors        test/parse-errors.sh soil-run-py             -
 runtime-errors      test/runtime-errors.sh run-all-with-osh      -
 oil-runtime-errors  test/oil-runtime-errors.sh run-all-with-osh  -

spec/stateful/interactive.py

@@ -28,7 +28,7 @@ def t9(sh):
 
 if __name__ == '__main__':
   try:
-    harness.main(sys.argv)
+    sys.exit(harness.main(sys.argv))
   except RuntimeError as e:
     print('FATAL: %s' % e, file=sys.stderr)
     sys.exit(1)

spec/stateful/job_control.py

@@ -67,7 +67,7 @@ def t8(sh):
 
 if __name__ == '__main__':
   try:
-    harness.main(sys.argv)
+    sys.exit(harness.main(sys.argv))
   except RuntimeError as e:
     print('FATAL: %s' % e, file=sys.stderr)
     sys.exit(1)

spec/stateful/signals.py

@@ -292,7 +292,7 @@ def t5(sh):
 
 if __name__ == '__main__':
   try:
-    harness.main(sys.argv)
+    sys.exit(harness.main(sys.argv))
   except RuntimeError as e:
     print('FATAL: %s' % e, file=sys.stderr)
     sys.exit(1)

test/spec-runner.sh

@@ -96,7 +96,7 @@ manifest() {
   #ls -l _tmp/spec/SUITE-*.txt
 }
 
-run-cases() {
+run-file() {
   local spec_name=$1
 
   log "__ $spec_name"
@@ -321,7 +321,7 @@ _all-parallel() {
 
   # The exit codes are recorded in files for html-summary to aggregate.
   set +o errexit
-  head -n $NUM_SPEC_TASKS $manifest | xargs -n 1 -P $MAX_PROCS -- $0 run-cases
+  head -n $NUM_SPEC_TASKS $manifest | xargs -n 1 -P $MAX_PROCS -- $0 run-file
   set -o errexit
 
   #ls -l _tmp/spec
@@ -336,7 +336,7 @@ _all-parallel() {
 all-parallel() {
   ### Run spec tests in parallel.
 
-  # Note that this function doesn't fail because 'run-cases' saves the status
+  # Note that this function doesn't fail because 'run-file' saves the status
   # to a file.
 
   time $0 _all-parallel "$@"

test/stateful.sh

@@ -15,26 +15,17 @@ set -o nounset
 set -o pipefail
 set -o errexit
 
-export PYTHONPATH=.
-
-readonly OSH=bin/osh
+source test/common.sh  # run-task-with-status
 
 # This uses ../oil_DEPS/spec-bin/{bash,dash} if they exist
-source build/dev-shell.sh
-
-# Use system shells until the Soil CI container has spec-bin
 
 # The ovm-tarball container that has spec-bin doesn't have python3 :-(  Really
 # we should build another container
-readonly BASH=bash
-readonly DASH=dash
-
-run() {
-  ### Wrapper for PYTHONPATH
+source build/dev-shell.sh
 
-  spec/stateful/signals.py "$@"
+export PYTHONPATH=.
 
-}
+readonly BASE_DIR=_tmp/stateful
 
 signals() {
   spec/stateful/signals.py --osh-failures-allowed 1 \
@@ -51,15 +42,107 @@ job-control() {
     $OSH bash "$@"
 }
 
+manifest() {
+  ### List all tests
+
+  cat <<EOF
+interactive
+job-control
+signals
+EOF
+}
+
+run-file() {
+  local spec_name=$1
+
+  log "__ $spec_name"
+
+  # could be 'test/spec-alpine.sh run-test', which WILL BE SPLIT!
+  local spec_runner=${SPEC_RUNNER:-test/spec.sh}
+  local base_dir=$BASE_DIR
+
+  # TODO: Could --stats-{file,template} be a separate awk step on .tsv files?
+  run-task-with-status \
+    $base_dir/${spec_name}.task.txt \
+    $0 $spec_name | tee $base_dir/${spec_name}.log.txt
+}
+
+html-summary() {
+  html-head --title 'Stateful Tests' \
+    ../../web/base.css ../../web/spec-tests.css
+
+  # Similar to test/spec-runner.sh and soil format-wwz-index
+
+  cat <<EOF
+  <body class="width50">
+
+<p id="home-link">
+  <a href="..">Up</a> |
+  <a href="/">oilshell.org</a>
+</p>
+
+    <h1>Stateful Tests with <a href="//www.oilshell.org/cross-ref.html#pexpect">pexpect</a> </h1>
+
+    <table>
+      <thead>
+        <tr>
+          <td>Task</td>
+          <td>Elapsed</td>
+          <td>Status</td>
+        </tr>
+      </thead>
+EOF
+
+  local all_passed=0
+
+  shopt -s lastpipe  # to mutate all_passed in while
+
+  manifest | while read spec_name; do
+    read status elapsed < $BASE_DIR/${spec_name}.task.txt
+    echo '<tr>'
+    echo "<td> <a href="$spec_name.log.txt">$spec_name</a> </td>"
+    echo "<td>$elapsed</td>"
+
+    case $status in
+      (0)  # exit code 0 is success
+        echo "  <td>$status</td>"
+        ;;
+      (*)  # everything else is a failure
+        # Add extra text to make red stand out.
+        echo "  <td class=\"fail\">status: $status</td>"
+
+        # Mark failure
+        all_passed=1
+        ;;
+    esac
+
+    echo '</tr>'
+  done
+
+  cat <<EOF
+    </table>
+  </body>
+</html>
+EOF
+
+  log "all_passed = $all_passed"
+
+  return $all_passed
+}
+
 all() {
   ### Run all tests
 
+  mkdir -p $BASE_DIR
+
   # TODO: The reports for each file should be written and uploaded.
-  # Can we reuse the test/spec table?
 
-  signals
-  interactive
-  job-control
+  set +o errexit
+  manifest | xargs -n 1 -- $0 run-file
+  set -o errexit
+
+  # Returns whether all passed
+  html-summary > $BASE_DIR/index.html
 }
 
 soil-run() {