Publish wild test results as a .wwz archive.

Most JS and CSS is outside of the archive for speed reasons. osh-to-oil.{js,html} is inside so relative URLs work on local disk and through a server. Also: - Change name from listing.html -> index.html. - Set the default sort order of both tables.
oilshell · Oct 15, 2017 · daa52c4fa5f2004797ad4f7448fc80a4b4b1ea0b · daa52c4
1 parent 09ee4a6
commit daa52c4fa5f2004797ad4f7448fc80a4b4b1ea0b
Unified Split

Showing with 135 additions and 35 deletions.

+46 −8 test/publish.sh

+15 −3 test/wild-runner.sh

+11 −3 test/wild.sh

+63 −21 test/wild_report.py
diff --git a/test/publish.sh b/test/publish.sh
@@ -7,18 +7,30 @@ set -o nounset
 set -o pipefail
 set -o errexit

-spec() {
-  local user=$1
-  local host=$2
-  local label=${3:-$(hostname)}  # what machine we ran it on
+log() {
+  echo "$@" 1>&2
+}
+
+versioned-dest() {
+  local label=${1:-$(hostname)}  # what machine we ran it on

  local branch=$(git rev-parse --abbrev-ref HEAD)
-  echo $branch
+  log "branch $branch"
  local hash=$(git rev-parse $branch)
  local short_hash=${hash:0:8}
-  echo $short_hash
+  log "hash $short_hash"
+
+  local dest="oilshell.org/git-branch/$branch/$short_hash/$label"
+  echo $dest
+}
+
+spec() {
+  local user=$1
+  local host=$2
+
+  local dest
+  dest="$(versioned-dest)/spec"

-  local dest=oilshell.org/git-branch/$branch/$short_hash/$label/spec
  ssh $user@$host mkdir -p $dest

  # rsync is faster than scp and has the --no-target-directory behavior.
@@ -37,8 +49,34 @@ unit() {
 }

 # Wild Tests.  NOTE: There's code for this in the oilshell.org repo.
+compress-wild() {
+  local dest=_tmp/wild-deploy
+  mkdir -p $dest
+
+  local out=$PWD/$dest/wild.wwz  # abs path for zip
+  rm -f -v $out
+
+  pushd _tmp/wild/www
+  time zip -r -q $out .  # recursive, quiet
+  popd
+
+  test/wild-runner.sh link-static $dest
+  ls -l -h $dest
+}
+
 wild() {
-  echo 'Hello from publish.sh'
+  local user=$1
+  local host=$2
+
+  local dest
+  dest="$(versioned-dest)"  # no wild/ suffix, since it's wild.wwz/
+
+  ssh $user@$host mkdir -p $dest
+
+  rsync --archive --verbose --copy-links \
+    _tmp/wild-deploy/ $user@$host:$dest/
+
+  echo "Visit http://$dest/wild.wwz/"
 }

 "$@"
diff --git a/test/wild-runner.sh b/test/wild-runner.sh
@@ -85,15 +85,27 @@ _link() {
  ln -s -f -v "$@"
 }

-make-report() {
-  print-manifest | wild-report summarize-dirs
+link-static() {
+  local dest=${1:-_tmp/wild}
+  local dest_ajax=${1:-_tmp/wild/www}

  _link \
    $PWD/web/wild.css \
-    $PWD/web/osh-to-oil.{html,js,css} \
    $PWD/web/ajax.js \
    $PWD/web/table/table-sort.{js,css} \
+    $dest
+}
+
+make-report() {
+  print-manifest | wild-report summarize-dirs
+
+  # This has to go inside the www dir because of the way that relative links
+  # are calculated.
+  _link \
+    $PWD/web/osh-to-oil.{html,js} \
    _tmp/wild/www
+
+  link-static
 }

 test-wild-report() {
diff --git a/test/wild.sh b/test/wild.sh
@@ -345,7 +345,12 @@ write-manifest() {
 # 760K lines without ltmain.sh.  Hm need to get up to 1M.

 abspaths() {
-  awk '{print $2}' _tmp/wild/MANIFEST.txt 
+  local proj=${1:-}
+  if test -n "$proj"; then
+    awk -v proj=$proj '$1 == proj {print $2}' _tmp/wild/MANIFEST.txt 
+  else
+    awk '{print $2}' _tmp/wild/MANIFEST.txt 
+  fi
 }

 # The biggest ones are all ltmain.sh though.
@@ -359,8 +364,11 @@ count-lines() {
    #grep -v ltmain.sh |
 }

-# Ideas: find shebang
-find-unicode() {
+# Takes ~15 seconds for 8,000+ files.
+#
+# NOTE: APKBUILD don't have shebang lines!  So there are a bunch of false
+# detections, e.g. APKBUILD as Makefile, C, etc.
+detect-all-types() {
  time abspaths | xargs file | pv > _tmp/wild/file-types.txt
 }

diff --git a/test/wild_report.py b/test/wild_report.py
@@ -51,17 +51,17 @@ def MakeHtmlGroup(title_str, body_str):
  <head>
    <title>{.template TITLE}</title>

-    <script type="text/javascript" src="{base_url}ajax.js"></script>
-    <script type="text/javascript" src="{base_url}table-sort.js"></script>
-    <link rel="stylesheet" type="text/css" href="{base_url}table-sort.css" />
-    <link rel="stylesheet" type="text/css" href="{base_url}wild.css" />
+    <script type="text/javascript" src="{base_url}../ajax.js"></script>
+    <script type="text/javascript" src="{base_url}../table-sort.js"></script>
+    <link rel="stylesheet" type="text/css" href="{base_url}../table-sort.css" />
+    <link rel="stylesheet" type="text/css" href="{base_url}../wild.css" />
  </head>

-  <body onload="initPage(gUrlHash, gTableStates, kStatusElem);"
+  <body onload="initPage(gUrlHash, gTables, gTableStates, kStatusElem);"
        onhashchange="onHashChange(gUrlHash, gTableStates, kStatusElem);">
    <p id="status"></p>

-    <p style="text-align: right"><a href="//www.oilshell.org/">oilshell.org</a></p>
+    <p style="text-align: right"><a href="/">oilshell.org</a></p>
 {.template NAV}

 {.template BODY}
@@ -185,7 +185,7 @@ def MakeHtmlGroup(title_str, body_str):
      {.end}

      <td class="name">
-        <a href="{name|htmltag}/listing.html">{name|html}/</a>
+        <a href="{name|htmltag}/index.html">{name|html}/</a>
      </td>
    </tr>
  {.end}
@@ -294,13 +294,30 @@ def MakeHtmlGroup(title_str, body_str):
  var gTableStates = {};
  var kStatusElem = document.getElementById('status');

-  function initPage(urlHash, tableStates, statusElem) {
-    var e1 = document.getElementById('dirs');
-    var e2 = document.getElementById('files');
-    var t = [];
-    if (e1) { t.push(e1); }
-    if (e2) { t.push(e2); }
-    makeTablesSortable(urlHash, t, tableStates);
+  var gTables = [];
+  var e1 = document.getElementById('dirs');
+  var e2 = document.getElementById('files');
+
+  // If no hash, "redirect" to a state where we sort ascending by dir name and
+  // filename.  TODO: These column numbers are a bit fragile.
+  var params = [];
+  if (e1) {
+    gTables.push(e1);
+    params.push('t:dirs=8a');
+  }
+  if (e2) {
+    gTables.push(e2);
+    params.push('t:files=7a');
+  }
+
+  function initPage(urlHash, gTables, tableStates, statusElem) {
+    makeTablesSortable(urlHash, gTables, tableStates);
+    /* Disable for now, this seems odd?  Think about mutability of gUrlHash.
+    if (location.hash === '') {
+      document.location = '#' + params.join('&');
+      gUrlHash = new UrlHash(location.hash);
+    }
+    */
    updateTables(urlHash, tableStates, statusElem);
  }

@@ -399,8 +416,8 @@ def DebugPrint(node, indent=0):


 def WriteJsonFiles(node, out_dir):
-  """Write a listing.json file for every directory."""
-  path = os.path.join(out_dir, 'INDEX.json')
+  """Write a index.json file for every directory."""
+  path = os.path.join(out_dir, 'index.json')
  with open(path, 'w') as f:
    raise AssertionError  # fix dir_totals
    d = {'files': node.files, 'dirs': node.dir_totals}
@@ -423,17 +440,42 @@ def _MakeNav(rel_path):
    if i == n - 1:
      link = None  # Current page shouldn't have link
    else:
-      link = '../' * (n - 1 - i) + 'listing.html'
+      link = '../' * (n - 1 - i) + 'index.html'
    data.append({'anchor': p, 'link': link})
  return data


+def _Lower(s):
+  return s.lower()
+
+
 def WriteHtmlFiles(node, out_dir, rel_path='', base_url=''):
-  """Write a listing.html file for every directory."""
-  path = os.path.join(out_dir, 'listing.html')
+  """Write a index.html file for every directory.
+
+  NOTE:
+  - osh-to-oil.html lives at $base_url
+  - table-sort.js lives at $base_url/../table-sort.js
+
+  wild/
+    table-sort.js
+    table-sort.css
+    www/
+      index.html
+      osh-to-oil.html
+
+  wild/
+    table-sort.js
+    table-sort.css
+    wild.wwz/  # Zip file
+      index.html
+      osh-to-oil.html
+
+  wwz latency is subject to caching headers.
+  """
+  path = os.path.join(out_dir, 'index.html')
  with open(path, 'w') as f:
    files = []
-    for name in sorted(node.files):
+    for name in sorted(node.files, key=_Lower):
      stats = node.files[name]
      entry = dict(stats)
      entry['name'] = name
@@ -443,7 +485,7 @@ def WriteHtmlFiles(node, out_dir, rel_path='', base_url=''):
      files.append(entry)

    dirs = []
-    for name in sorted(node.dirs):
+    for name in sorted(node.dirs, key=_Lower):
      entry = dict(node.dirs[name].subtree_stats)
      entry['name'] = name
      # TODO: This should be internal time