Miscellaneous tweaks to make the wild.sh report look and work better.

- Add a top-level subtree summary to each WILD directory entry.  This
  required attaching subtree stats to each DirNode.

- Calculate and show the maximum parse time.
  - Currently 18 seconds for the 9,000 line ltmain.sh.  But this should
    be multiplied by 10 because I have a really fast computer!  180
    seconds needs to be less than 180 ms.  Three orders of magnitude.
- Calculate and show the maximum number of lines.

- script script selection tweaks
  - Add Kythe
  - Exclude non-shell scripts from pixelb and git (nice use of find
    -exec test/shebang.sh is-shell {} ';')

- Unify CSS: Use table-sort.css in the generated listing.html files.

- Functions to look for unicode chars

test/shebang.sh

@@ -0,0 +1,26 @@
#!/bin/bash
#
# Usage:
#   ./shebang-is-shell.sh PATH

# Test if the first line ends with 'sh'.
is-shell() {
  local path=$1
  local shebang
  read shebang < $path  # read a line from the file
  shebang=${shebang// /}  # strip all whitespace on the line
  [[ $shebang == *sh ]]
}

unittest() {
  for file in bin/oil.py configure install; do
    if is-shell $file; then
      echo YES $file
    else
      echo NO $file
    fi
  done
}

"$@"

test/wild-runner.sh

@@ -50,14 +50,11 @@ process-file() {
readonly NUM_TASKS=200

print-manifest() {
  #head _tmp/wild/MANIFEST.txt 
  #egrep '^dokku|^wwwoosh|^oil' _tmp/wild/MANIFEST.txt
  #head -n $NUM_TASKS _tmp/wild/MANIFEST.txt
  #egrep -- 'mesos' _tmp/wild/MANIFEST.txt
  #egrep '^dokku|^wwwoosh|^oil' _tmp/wild/MANIFEST.txt
  #egrep -- '^pixelb' _tmp/wild/MANIFEST.txt
  #egrep -- '^oil' _tmp/wild/MANIFEST.txt
  cat _tmp/wild/MANIFEST.txt
  #egrep -- '^aports' _tmp/wild/MANIFEST.txt
  #egrep -- '^gherkin' _tmp/wild/MANIFEST.txt
  #egrep -- '^bash' _tmp/wild/MANIFEST.txt
}

parse-all() {
@@ -95,10 +92,14 @@ make-report() {
    $PWD/web/wild.css \
    $PWD/web/osh-to-oil.{html,js,css} \
    $PWD/web/ajax.js \
    $PWD/web/table/table-sort.js \
    $PWD/web/table/table-sort.{js,css} \
    _tmp/wild/www
}

test-wild-report() {
  egrep -- '^oil|^perf-tools' _tmp/wild/MANIFEST.txt | wild-report summarize-dirs
}

if test "$(basename $0)" = 'wild-runner.sh'; then
  "$@"
fi

test/wild.sh

@@ -176,7 +176,14 @@ all-manifests() {
  _sh-manifest ~/src/linux-4.8.7

  # Git
  _sh-manifest ~/git/other/git
  # git-gui.sh and po2msg.sh are actually Tcl!  We could stop parsing at 'exec'
  # but there's no point right now.
  src=~/git/other/git
  _manifest $(basename $src) $src \
    $(find $src -name '*.sh' -a \
      ! -name 'git-gui.sh' \
      ! -name 'po2msg.sh' \
      -a -printf '%P\n')

  #
  # Cloud Stuff
@@ -195,6 +202,7 @@ all-manifests() {
  #
  _sh-manifest ~/git/other/bazel
  _sh-manifest ~/git/other/protobuf
  _sh-manifest ~/git/other/kythe

  #
  # Other shells
@@ -277,12 +285,19 @@ all-manifests() {
  # Misc Scripts
  #

  # NOTE: These scripts don't end with *.sh
  # Most of these scripts have no extension.  So look at executable ones and
  # then see if the shebang ends with sh!

  # NOTE: In Oil it would be nice if shebang-is-shell could be a function call.
  # Don't need to fork every time.
  src=~/git/other/pixelb-scripts
  _manifest pixelb-scripts $src \
    $(find $src \( -name .git -a -prune \) -o \
                \( -type f -a -executable -a -printf '%P\n' \) )

                \( -type f -a \
                   -executable -a \
                   ! -name '*.py' -a \
                   -exec test/shebang.sh is-shell {} ';' -a \
                   -printf '%P\n' \) )

  # Something related to WebDriver
  # Doesn't parse because of extended glob.
@@ -329,17 +344,30 @@ write-manifest() {
# 1.30 M lines with "big".
# 760K lines without ltmain.sh.  Hm need to get up to 1M.

abspaths() {
  awk '{print $2}' _tmp/wild/MANIFEST.txt 
}

# The biggest ones are all ltmain.sh though.
count-lines() {
  # We need this weird --files0-from because there are too many files.  xargs
  # would split it into multiple invocations.
  #
  # It would be nicer if wc just had an option not to sum?
  time awk '{print $2}' _tmp/wild/MANIFEST.txt | 
  time abspaths | 
    tr '\n' '\0' | wc -l --files0-from - | sort -n
    #grep -v ltmain.sh |
}

# Ideas: find shebang
find-unicode() {
  time abspaths | xargs file | pv > _tmp/wild/file-types.txt
}

wild-types() {
  cat _tmp/wild/file-types.txt | test/wild_types.py
}

all() {
  test/wild-runner.sh all-parallel "$@"
}