Completely rewrite the test/wild.sh framework.

Prior to this change, it ran parsing and translation serially, and produced HTML with shell. There was a separate FILES.html and FAILED.html index for each project. Now it does everything in parallel, and produces HTML with Python. There is a unified listing.html for each subdirectory (not each project). Details: test/common.sh: Factored out functions common to test/{spec,wild}.sh. test/wild-runner.sh: New driver to run all OSH tasks in parallel. test/wild.sh: Greatly simplify the selection of shell scripts with 'find'. Instead of separate functions, use a single function to write _tmp/wild/MANIFEST.txt. test/wild_report.py: Walk the _tmp/wild/raw tree and produce the _tmp/wild/www tree. TODO: Vendor in JSON Template dependency. Unrelated: - Remove obsolete notes from osh/osh.asdl
oilshell · Oct 12, 2017 · b567c492660b2a86c950f76e563c4ad5ef43a20f · b567c49
1 parent b01362d
commit b567c492660b2a86c950f76e563c4ad5ef43a20f
diff --git a/osh/osh.asdl b/osh/osh.asdl
@@ -11,8 +11,7 @@
 --   * hidden behind set -o extglob
 -- * let arithmetic (rarely used)
 -- * coprocesses -- one with arg and one without
-- * Append operator in Assignment: +=
-- * &> redirect stdout and stderr
+-- * &> redirect both stdout and stderr
 -- * 1>&2- to close redirect

 -- Represented but Not Parsed:
@@ -29,7 +28,6 @@
 -- * parens
 --   * 1 + 2*3  vs.  1 + (2*3)  or even 1 + ((2*3))
 --   * [[ (1 == 1) ]] vs [[ 1 == 1 ]]
-- * HereDoc vs HereWord.  I collapsed them.
 -- * $(( 1 + 2 )) vs $[1 + 2]  (bash-specific, used by Aboriginal)
 --
 -- Found to be not strictly necessary for oil conversion
diff --git a/test/common.sh b/test/common.sh
@@ -0,0 +1,51 @@
+#!/bin/bash
+#
+# Usage:
+#   ./common.sh <function name>
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+# For xargs -P in spec-runner.sh, wild-runner.sh.
+
+readonly JOBS=$(( $(nproc) - 1 ))
+
+log() {
+  echo "$@" 1>&2
+}
+
+die() {
+  log "$@"
+  exit 1
+}
+
+run-task-with-status() {
+  local out_file=$1
+  shift
+
+  # --quiet suppresses a warning message
+  /usr/bin/env time \
+    --output $out_file \
+    --format '%x %e' \
+    -- "$@" || true  # suppress failure
+
+  # Hack to get around the fact that --quiet is Debian-specific:
+  # http://lists.oilshell.org/pipermail/oil-dev-oilshell.org/2017-March/000012.html
+  #
+  # Long-term solution: our xargs should have --format.
+  sed -i '/Command exited with non-zero status/d' $out_file
+
+  # TODO: Use rows like this with oil
+  # '{"status": %x, "wall_secs": %e, "user_secs": %U, "kernel_secs": %S}' \
+}
+
+run-task-with-status-test() {
+  run-task-with-status _tmp/status.txt sh -c 'sleep 0.1; exit 1' || true
+  cat _tmp/status.txt
+  test "$(wc -l < _tmp/status.txt)" = '1' || die "Expected only one line"
+}
+
+if test "$(basename $0)" = 'common.sh'; then
+  "$@"
+fi
diff --git a/test/spec-runner.sh b/test/spec-runner.sh
@@ -9,10 +9,7 @@ set -o nounset
 set -o pipefail
 set -o errexit

-die() {
-  echo 1>&2 "$@"
-  exit 1
-}
+source test/common.sh

 #
 # Test Runner
@@ -57,32 +54,6 @@ run-cases() {
    > _tmp/spec/${spec_name}.html
 }

-run-task-with-status() {
-  local out_file=$1
-  shift
-
-  # --quiet suppresses a warning message
-  /usr/bin/env time \
-    --output $out_file \
-    --format '%x %e' \
-    -- "$@" || true  # suppress failure
-
-  # Hack to get around the fact that --quiet is Debian-specific:
-  # http://lists.oilshell.org/pipermail/oil-dev-oilshell.org/2017-March/000012.html
-  #
-  # Long-term solution: our xargs should have --format.
-  sed -i '/Command exited with non-zero status/d' $out_file
-
-  # TODO: Use rows like this with oil
-  # '{"status": %x, "wall_secs": %e, "user_secs": %U, "kernel_secs": %S}' \
-}
-
-run-task-with-status-test() {
-  run-task-with-status _tmp/status.txt sh -c 'sleep 0.1; exit 1' || true
-  cat _tmp/status.txt
-  test "$(wc -l < _tmp/status.txt)" = '1' || die "Expected only one line"
-}
-
 readonly NUM_TASKS=400
 #readonly NUM_TASKS=4

@@ -236,8 +207,6 @@ link-css() {
  ln -s -f --verbose $PWD/web/{spec-tests,spec-code}.css _tmp/spec
 }

-readonly JOBS=$(( $(nproc) - 1 ))
-
 _all-parallel() {
  mkdir -p _tmp/spec

diff --git a/test/unit.sh b/test/unit.sh
@@ -14,7 +14,7 @@ set -o nounset
 set -o pipefail
 set -o errexit

-source test/spec-runner.sh  # TODO: Separate this?
+source test/common.sh

 export PYTHONPATH=.  # current dir

diff --git a/test/wild-runner.sh b/test/wild-runner.sh
@@ -0,0 +1,101 @@
+#!/bin/bash
+#
+# Usage:
+#   ./wild-runner.sh <function name>
+
+set -o nounset
+set -o pipefail
+set -o errexit
+
+source test/common.sh
+
+process-file() {
+  local proj=$1
+  local abs_path=$2
+  local rel_path=$3
+
+  local raw_base=_tmp/wild/raw/$proj/$rel_path
+  local www_base=_tmp/wild/www/$proj/$rel_path
+  mkdir -p $(dirname $raw_base)
+  mkdir -p $(dirname $www_base)
+
+  log "--- Processing $proj - $rel_path"
+
+  # Count the number of lines.  This creates a tiny file, but we're doing
+  # everything involving $abs_path at once so it's in the FS cache.
+  wc $abs_path > ${raw_base}__wc.txt
+
+  # Make a literal copy with .txt extension, so we can browse it
+  cp $abs_path ${www_base}.txt
+
+  # Parse the file.
+  local task_file=${raw_base}__parse.task.txt
+  local stderr_file=${raw_base}__parse.stderr.txt
+  local out_file=${www_base}__ast.html
+
+  run-task-with-status $task_file \
+    bin/osh --ast-format abbrev-html -n $abs_path \
+    > $out_file 2> $stderr_file
+
+  # Convert the file.
+  task_file=${raw_base}__osh2oil.task.txt
+  stderr_file=${raw_base}__osh2oil.stderr.txt
+  out_file=${www_base}__oil.txt
+
+  run-task-with-status $task_file \
+    bin/osh -n --fix $abs_path \
+    > $out_file 2> $stderr_file
+}
+
+readonly NUM_TASKS=200
+
+print-manifest() {
+  #head _tmp/wild/MANIFEST.txt 
+  #egrep '^dokku|^wwwoosh|^oil' _tmp/wild/MANIFEST.txt
+  #head -n $NUM_TASKS _tmp/wild/MANIFEST.txt
+  #egrep -- 'mesos' _tmp/wild/MANIFEST.txt
+  #cat _tmp/wild/MANIFEST.txt
+  #egrep -- '^aports' _tmp/wild/MANIFEST.txt
+  egrep -- '^gherkin' _tmp/wild/MANIFEST.txt
+}
+
+parse-all() {
+  local failed=''
+  #head -n 20 _tmp/wild/MANIFEST.txt |
+  print-manifest | xargs -n 3 -P $JOBS -- $0 process-file || failed=1
+
+  # Limit the output depth
+  tree -L 3 _tmp/wild
+}
+
+# Takes 3m 47s on 7 cores for 513K lines.
+# So that's like 230 seconds or so.  It should really take 1 second!
+
+all-parallel() {
+  time {
+    test/wild.sh write-manifest
+    parse-all
+    make-report
+  }
+}
+
+wild-report() {
+  PYTHONPATH=~/hg/json-template/python test/wild_report.py "$@";
+}
+
+_link() {
+  ln -s -f -v "$@"
+}
+
+make-report() {
+  print-manifest | wild-report summarize-dirs
+
+  _link \
+    $PWD/web/wild.css \
+    $PWD/web/osh-to-oil.{html,js,css} \
+    _tmp/wild/www
+}
+
+if test "$(basename $0)" = 'wild-runner.sh'; then
+  "$@"
+fi