Add many more projects to wild tests.

e.g. the source code to the book Wicked Cool Shell Scripts. - Also add the ability to filter wild test execution by project, but report on all. Unrelated: - In the OSH quick ref, Put the 'select' language construct in the right place. - Fix log messages in quick_ref.py.
oilshell · Oct 17, 2017 · 8d245263a1b64bd243528964253c0ff63e3b1c79 · 8d24526
1 parent 2155281
commit 8d245263a1b64bd243528964253c0ff63e3b1c79
Unified Split

Showing with 77 additions and 32 deletions.

+5 −0 build/doc.sh

+3 −4 build/quick_ref.py

+2 −2 doc/osh-quick-ref-toc.txt

+1 −0 osh/osh.asdl

+16 −18 test/wild-runner.sh

+50 −8 test/wild.sh
diff --git a/build/doc.sh b/build/doc.sh
@@ -37,6 +37,10 @@ set -o errexit
 readonly OIL_VERSION=$(head -n 1 oil-version.txt)
 export OIL_VERSION  # for quick_ref.py

+log() {
+  echo "$@" 1>&2
+}
+
 _build-timestamp() {
  echo '<hr/>'
  echo "<i>Generated on $(date)</i>"
@@ -110,6 +114,7 @@ EOF
 </html>
 EOF
  } > $html_out
+  log "Wrote $html_out"
 }

 osh-quick-ref() {
diff --git a/build/quick_ref.py b/build/quick_ref.py
@@ -308,10 +308,10 @@ def main(argv):
      TableOfContents(f)

  elif action == 'pages':
-    html_out, text_dir, py_out_path = argv[2:5]
+    pages_txt, text_dir, py_out_path = argv[2:5]

    topic_lookup = {}
-    with open(html_out) as f:
+    with open(pages_txt) as f:
      text_out = TextOutput(text_dir, topic_lookup)
      Pages(f, text_out)

@@ -322,8 +322,7 @@ def main(argv):
      f.write('TOPIC_LOOKUP = ')
      f.write(d)

-    print >>sys.stderr, 'Wrote %s, %s, and %s' % (
-        html_out, text_dir, py_out_path)
+    print >>sys.stderr, 'Wrote %s/ and %s' % (text_dir, py_out_path)

  else:
    raise RuntimeError('Invalid action %r' % action)
diff --git a/doc/osh-quick-ref-toc.txt b/doc/osh-quick-ref-toc.txt
@@ -21,7 +21,7 @@ COMMAND LANGUAGE
  [Redirects]     redir-file  >  >>  >|  <  <>   X &>
                  redir-desc  >&  <&
                  here-doc    <<  <<-  <<<
-  [Other]         dparen ((   time   coproc
+  [Other]         dparen ((   time   X coproc   X select

 ASSIGNING VARIABLES
  [Keywords]      local   readonly   export   unset   shift
@@ -59,7 +59,7 @@ OTHER SHELL SUBLANGUAGES

 BUILTIN COMMANDS
  [I/O]           read   echo 
-                  X readarray   X mapfile   X select
+                  X readarray   X mapfile
  [Run Code]      source .   eval
  [Set Options]   set   shopt
  [Working Dir]   cd   pwd   pushd   popd   dirs
diff --git a/osh/osh.asdl b/osh/osh.asdl
@@ -11,6 +11,7 @@
 --   * hidden behind set -o extglob
 -- * let arithmetic (rarely used)
 -- * coprocesses -- one with arg and one without
+-- * select block
 -- * &> redirect both stdout and stderr
 -- * 1>&2- to close redirect

diff --git a/test/wild-runner.sh b/test/wild-runner.sh
@@ -48,21 +48,11 @@ process-file() {
 }

 readonly NUM_TASKS=200
+readonly MANIFEST=_tmp/wild/MANIFEST.txt

-print-manifest() {
-  #head -n $NUM_TASKS _tmp/wild/MANIFEST.txt
-  #egrep '^dokku|^wwwoosh|^oil' _tmp/wild/MANIFEST.txt
-  #egrep -- '^pixelb' _tmp/wild/MANIFEST.txt
-  #egrep -- '^oil' _tmp/wild/MANIFEST.txt
-  #egrep -- '^oil' _tmp/wild/MANIFEST.txt
-  #egrep -- '^esoteric' _tmp/wild/MANIFEST.txt
-  cat _tmp/wild/MANIFEST.txt
-}
-
-parse-all() {
+parse-in-parallel() {
  local failed=''
-  #head -n 20 _tmp/wild/MANIFEST.txt |
-  print-manifest | xargs -n 3 -P $JOBS -- $0 process-file || failed=1
+  xargs -n 3 -P $JOBS -- $0 process-file || failed=1

  # Limit the output depth
  tree -L 3 _tmp/wild
@@ -71,16 +61,24 @@ parse-all() {
 # Takes 3m 47s on 7 cores for 513K lines.
 # So that's like 230 seconds or so.  It should really take 1 second!

-all-parallel() {
+parse-and-report() {
+  local manifest_regex=${1:-}  # egrep regex for manifest line
+
  time {
    test/wild.sh write-manifest
-    parse-all
+
+    if test -n "$manifest_regex"; then
+      egrep -- "$manifest_regex" $MANIFEST | parse-in-parallel
+    else
+      cat $MANIFEST | parse-in-parallel
+    fi
+
    make-report
  }
 }

 wild-report() {
-  PYTHONPATH=~/hg/json-template/python test/wild_report.py "$@";
+  PYTHONPATH=~/hg/json-template/python test/wild_report.py "$@"
 }

 _link() {
@@ -99,7 +97,7 @@ link-static() {
 }

 make-report() {
-  print-manifest | wild-report summarize-dirs
+  cat $MANIFEST | wild-report summarize-dirs

  # This has to go inside the www dir because of the way that relative links
  # are calculated.
@@ -111,7 +109,7 @@ make-report() {
 }

 test-wild-report() {
-  egrep -- '^oil|^perf-tools' _tmp/wild/MANIFEST.txt | wild-report summarize-dirs
+  egrep -- '^oil|^perf-tools' $MANIFEST | wild-report summarize-dirs
 }

 if test "$(basename $0)" = 'wild-runner.sh'; then
diff --git a/test/wild.sh b/test/wild.sh
@@ -16,8 +16,6 @@ set -o nounset
 set -o pipefail
 set -o errexit

-source test/wild-runner.sh
-
 readonly RESULT_DIR=_tmp/wild

 #
@@ -91,10 +89,23 @@ all-manifests() {
  # Don't expose this repo for now
  #oil-sketch-manifest

+  #
+  # My Code
+  #
+
  oil-manifest

  local src

+  #
+  # Books
+  #
+  src=~/git/wild/books/wicked_cool_shell_scripts_2e
+  _manifest "books/$(basename $src)" $src \
+    $(find $src \
+      \( -name .git -a -prune \) -o \
+      \( -type f -a -executable -a -printf '%P\n' \) )
+
  #
  # Shell/Bash Frameworks/Collections
  #
@@ -136,8 +147,12 @@ all-manifests() {
      \( -name .git -a -prune \) -o \
      \( -type f -a -executable -a -printf '%P\n' \) )

+  _sh-manifest ~/git/other/modernish shell
+  _sh-manifest ~/git/other/posixcube shell
+
  # Shells themselves
  _sh-manifest ~/git/other/ast shell  # korn shell stuff
+  _sh-manifest ~/git/other/mwc-sh shell
  _sh-manifest ~/src/mksh shell

  #
@@ -146,6 +161,7 @@ all-manifests() {

  _sh-manifest ~/git/other/minimal distro
  _sh-manifest ~/git/other/linuxkit distro
+  _sh-manifest ~/git/other/portage distro

  src=~/git/alpine/aports
  _manifest distro/alpine-aports $src \
@@ -169,13 +185,24 @@ all-manifests() {
    $(find $src '(' -name debootstrap -o -name functions ')' -a -printf '%P\n') \
    $(find $src/scripts -type f -a -printf 'scripts/%P\n')

+  #
+  # Operating Systems
+  #
+
+  _sh-manifest ~/git/other/minix
+  _sh-manifest ~/git/other/illumos-gate
+  _sh-manifest ~/git/other/daemontools-encore
+
  #
  # Cloud Stuff
  #
  _sh-manifest ~/git/other/mesos cloud
+  _sh-manifest ~/git/other/rocket cloud
+  _sh-manifest ~/git/other/docker cloud
  _sh-manifest ~/git/other/chef-bcpc cloud
  _sh-manifest ~/git/other/sandstorm cloud
  _sh-manifest ~/git/other/kubernetes cloud
+  _sh-manifest ~/git/other/manta-marlin cloud

  src=~/git/other/dokku
  _manifest cloud/dokku $src \
@@ -187,6 +214,9 @@ all-manifests() {
  _sh-manifest ~/git/other/bazel google
  _sh-manifest ~/git/other/protobuf google
  _sh-manifest ~/git/other/kythe google
+  _sh-manifest ~/git/other/tensorflow google
+  # Filenames with spaces!
+  #_sh-manifest ~/git/other/grpc google

  #
  # Esoteric
@@ -233,6 +263,7 @@ all-manifests() {
  #

  _sh-manifest ~/git/other/julia
+  _sh-manifest ~/git/other/reason
  _sh-manifest ~/git/other/sdk  # Dart SDK?

  _sh-manifest ~/git/other/micropython
@@ -271,10 +302,17 @@ all-manifests() {
      ! -name 'po2msg.sh' \
      -a -printf '%P\n')

+  _sh-manifest ~/git/other/liballocs
+  _sh-manifest ~/git/other/boringssl
+  _sh-manifest ~/git/other/arrow
+
  #
  # Uncategorized
  #

+  # Has filenames with spaces!
+  #_sh-manifest ~/git/other/linguist
+
  # Brendan Gregg's performance scripts.
  # Find executable scripts, since they don't end in sh.
  # net/tcpretrans is written in Perl.
@@ -329,9 +367,12 @@ all-manifests() {
  }
 }

+# TODO: Parameterize this; it's duplicated in wild-runner.txt.
+readonly MANIFEST=_tmp/wild/MANIFEST.txt
+
 write-manifest() {
  mkdir -p _tmp/wild
-  local out=_tmp/wild/MANIFEST.txt
+  local out=$MANIFEST
  all-manifests > $out
  wc -l $out
 }
@@ -347,21 +388,21 @@ write-manifest() {
 abspaths() {
  local proj=${1:-}
  if test -n "$proj"; then
-    awk -v proj=$proj '$1 == proj {print $2}' _tmp/wild/MANIFEST.txt 
+    awk -v proj=$proj '$1 == proj {print $2}' $MANIFEST
  else
-    awk '{print $2}' _tmp/wild/MANIFEST.txt 
+    awk '{print $2}' $MANIFEST
  fi
 }

-# The biggest ones are all ltmain.sh though.
+# Excluding ltmain.sh, goes from 910K lines to 830K.
 count-lines() {
  # We need this weird --files0-from because there are too many files.  xargs
  # would split it into multiple invocations.
  #
  # It would be nicer if wc just had an option not to sum?
  time abspaths | 
+    grep -v ltmain.sh |
    tr '\n' '\0' | wc -l --files0-from - | sort -n
-    #grep -v ltmain.sh |
 }

 # Takes ~15 seconds for 8,000+ files.
@@ -376,8 +417,9 @@ wild-types() {
  cat _tmp/wild/file-types.txt | test/wild_types.py
 }

+# Make a report for all, but only run some
 all() {
-  test/wild-runner.sh all-parallel "$@"
+  test/wild-runner.sh parse-and-report "$@"
 }

 #