Make a new _release/VERSION tree for testing release the doc tree.

It has .wwz files for tests, benchmarks, and metrics. It has a web/ directory that mirrors $REPO_ROOT/web. The wild tests/spec tests/osh-parser benchmark now all share it.
oilshell · Nov 4, 2017 · 53951730b521776c9e684705c1972a4fe56f5a63 · 5395173
1 parent 9abab43
commit 53951730b521776c9e684705c1972a4fe56f5a63
diff --git a/INSTALL.txt b/INSTALL.txt
@@ -12,8 +12,8 @@ Quick Start

 If you haven't already done so, extract the tarball:
 
-    tar -x --xz < oil-0.1.0.tar.xz
-    cd oil-0.1.0
+    tar -x --xz < oil-0.2.alpha1.tar.xz
+    cd oil-0.2.alpha1

 Either install as /usr/local/bin/osh:

@@ -31,7 +31,7 @@ The latter doesn't require root access, but it requires ~/bin to be in your
 PATH.

 NOTE: Out-of-tree builds are NOT currently supported, so you have to be in the
-oil-0.1.0 directory.
+oil-0.2.alpha1 directory.

 Smoke Test
 ----------
diff --git a/README.md b/README.md
@@ -90,10 +90,27 @@ Directory Structure
    # Temporary Directories

    _bin/             # Native executables are put here
-    _release/         # Source release tarballs are put here
    _build/           # Temporary build files
    _devbuild/        # Developer build files not deleted upon 'make clean'
    _tmp/             # Temporary test files and the like
+      spec/
+      wild/
+        raw/
+        www/
+      osh-parser/
+      ...
+    _release/         # Source release tarballs are put here
+      VERSION/        # Published at oilshell.org/release/$VERSION/
+        benchmarks/
+        doc/
+        metrics/
+        test/
+          spec.wwz
+          wild.wwz
+          ...
+        web/          # Static files, copy of $REPO_ROOT/web
+          table/
+      

    # Dev Docs

diff --git a/benchmarks/osh-parser.sh b/benchmarks/osh-parser.sh
@@ -165,7 +165,7 @@ summarize() {
 # NOTE: not bothering to make it sortable now.  Just using the CSS.

 _print-report() {
-  local base_url='../../../web/table'
+  local base_url='../../web/table'

  cat <<EOF
 <!DOCTYPE html>
@@ -254,7 +254,8 @@ EOF
 }

 report() {
-  local out=$BASE_DIR/www/summary.html
+  local out=$BASE_DIR/index.html
+  mkdir -p $(dirname $out)
  _print-report > $out
  echo "Wrote $out"
 }
diff --git a/build/doc.sh b/build/doc.sh
@@ -61,7 +61,7 @@ _quick-ref() {
 x-quick-ref() {
  local prog=$1

-  local html_out=_build/doc/$prog-quick-ref.html
+  local html_out=_release/VERSION/doc/$prog-quick-ref.html
  local text_out_dir=_build/$prog-quick-ref
  local py_out=_devbuild/${prog}_help.py

@@ -82,7 +82,7 @@ x-quick-ref() {
      }
      body {
        margin: 0 auto;
-        width: 50em;
+        width: 40em;
      }
      /* different color because they're links but not topics */
      .level1 {
@@ -95,9 +95,15 @@ x-quick-ref() {
      h1,h2,h3,h4 {
      /* color: darkcyan; */
      }
+      #home-link {
+        text-align: right;
+      }
    </style>
  </head>
  <body>
+    <p id="home-link">
+      <a href="/">oilshell.org</a>
+    </p>
    <p style="color: darkred; font-size: x-large;">
      NOTE: This document is a work in progress!
    </p>
@@ -145,9 +151,15 @@ markdown2html() {
        color: green;
        margin-left: 4em;
      }
+      #home-link {
+        text-align: right;
+      }
    </style>
  </head>
  <body>
+    <p id="home-link">
+      <a href="/">oilshell.org</a>
+    </p>
 EOF
  
    markdown < $src  # TODO: CommonMark
@@ -163,7 +175,7 @@ EOF
 readonly MONOSPACE='font-family: monospace;'

 install() {
-  markdown2html INSTALL.txt _build/doc/INSTALL.html "$MONOSPACE"
+  markdown2html INSTALL.txt _release/VERSION/doc/INSTALL.html "$MONOSPACE"
 }

 release-index() {
diff --git a/doc/release-index.md b/doc/release-index.md
@@ -1,6 +1,6 @@
 <!-- NOTE: This file is at /release/$VERSION/index.html -->

-Oil Version 0.1.0
+Oil Version 0.2.alpha1
 -----------------

 ### What's New
@@ -24,6 +24,18 @@ Oil Version 0.1.0

 ### Links

- [Spec Test Results](test/spec/RESULTS.html).  OSH behavior is tested against
+- [Spec Test Results](test/spec.wwz/RESULTS.html).  OSH behavior is tested against
  the behavior of existing shells.
- [Source Code Metrics](metrics/).  How much code is in Oil?
+- [Wild Test Results](test/wild.wwz/).  Parsing thousands of shell scripts
+  with OSH.
+- [OSH Parser Performance](benchmarks/osh-parser.wwz/).  How fast does OSH parse
+  shell scripts?
+- [Line Counts](metrics/line-counts.wwz/).  How much code is in Oil?
+
+<!-- TODO: 
+benchmarks/
+  virtual-memory/
+src/                       annotated/cross-referenced source code
+coverage/                  code coverage in Python and C
+metrics/                   line-counts, debug info size?
+-->
diff --git a/scripts/release.sh b/scripts/release.sh
@@ -285,41 +285,142 @@ announcement-0.1() {
    > ../oilshell.org__deploy/release/$version/announcement.html
 }

-deploy-doc() {
-  local deploy_repo='../oilshell.org__deploy'
-  local release_root_dir="$deploy_repo/release"
-  local release_dir="$release_root_dir/$OIL_VERSION"
+_link() {
+  ln -s -f -v --no-target-directory "$@"
+}
+
+compress() {
+  local root=$PWD/_release/VERSION/
+
+  log "--- test/spec"
+  local out="$root/test/spec.wwz"
+  pushd _tmp/spec
+  time zip -r -q $out .  # recursive, quiet
+  popd
+
+  log "--- test/wild"
+  local out="$root/test/wild.wwz"
+
+  pushd _tmp/wild/www
+  time zip -r -q $out .  # recursive, quiet
+  popd
+
+  log "--- metrics/line-counts"
+  local out="$root/metrics/line-counts.wwz"
+  pushd _tmp/metrics/line-counts
+  time zip -r -q $out .  # recursive, quiet
+  popd
+
+  log "--- benchmarks/osh-parser"
+  local out="$root/benchmarks/osh-parser.wwz"
+  pushd _tmp/osh-parser/
+  time zip -r -q $out .  # recursive, quiet
+  popd
+
+  tree _release/VERSION
+}
+
+metrics-index() {
+  # TODO: Have to write index.html so it will serve out of .wwz.
+  cat <<EOF
+<!DOCTYPE html>
+<html>
+  <head>
+    <title>Line Counts</title>
+    <style>
+      body {
+        margin: 0 auto;
+        width: 40em;
+      }
+      #home-link {
+        text-align: right;
+      }
+    </style>
+  </head>
+  <body>
+    <p id="home-link">
+      <a href="/">oilshell.org</a>
+    </p>
+    <h3>Line Counts</h3>
+    <p>
+EOF
+  find $out -name '*.txt' -a -printf '<a href="%P">%P</a> <br/>'
+cat <<EOF
+    </p>
+  </body>
+</html>
+EOF
+}
+
+line-counts() {
+  local out=_tmp/metrics/line-counts
+  mkdir -p $out
+
+  # Metrics.  TODO: It would be nicer to make this structured data somehow.
+  scripts/count.sh all > $out/src.txt  # Count repo lines
+  build/metrics.sh linecount-pydeps > $out/pydeps.txt
+  build/metrics.sh linecount-nativedeps > $out/nativedeps.txt

-  mkdir -p $release_dir/{doc,test,metrics}
+  metrics-index > $out/index.html
+  tree $out
+}
+
+_copy-path() {
+  local src=$1 dest=$2
+  mkdir -p $(dirname $dest)
+  cp -v $src $dest
+}
+
+copy-web() {
+  find web \
+    \( -name _tmp -a -prune \) -o \
+    \( -name '*.css' -o -name '*.js' \) -a -printf '%p _release/VERSION/%p\n' |
+  xargs -n 2 -- $0 _copy-path
+}
+
+# TODO:
+# Test out web/ *.css,js,html
+# metrics/line-counts.wwz/
+#   src.txt
+#   pydeps.txt
+#   nativedeps.txt
+
+build-tree() {
+  local root=_release/VERSION
+  mkdir -p $root/{doc,test,benchmarks,metrics}

  # Metadata
-  cp -v _build/release-date.txt oil-version.txt $release_dir
+  cp -v _build/release-date.txt oil-version.txt $root
+
+  # Docs
+  build/doc.sh osh-quick-ref
+  build/doc.sh install
+  build/doc.sh release-index $root/index.html

-  # Line counts.  TODO: It would be nicer to make this structured data somehow.
-  scripts/count.sh all \
-    > $release_dir/metrics/linecount-src.txt  # Count repo lines
-  build/metrics.sh linecount-pydeps \
-    > $release_dir/metrics/linecount-pydeps.txt
-  build/metrics.sh linecount-nativedeps \
-    > $release_dir/metrics/linecount-nativedeps.txt
+  # Problem: You can't preview it without .wwz!
+  # Maybe have local redirects VERSION/test/wild/ to 
+  #
+  # Instead of linking, I should compress them all here.
+
+  copy-web
+
+  tree $root
+}
+
+deploy-doc() {
+  local deploy_repo='../oilshell.org__deploy'
+  local release_root_dir="$deploy_repo/release"
+  local release_dir="$release_root_dir/$OIL_VERSION"

  # --dereference because test/spec has css symlinks that should be files.
  # rsync should just transfer raw files.
  cp -v -r --force --dereference --no-target-directory \
-    _tmp/spec/ $release_dir/test/spec
+    _release/VERSION/ $release_dir/

  # Generate release index.
  html-index $release_root_dir _tmp/releases.html
  cp -v _tmp/releases.html $deploy_repo

-  build/doc.sh osh-quick-ref
-  # Generate docs.
-  build/doc.sh install
-
-  cp -v -r --no-target-directory _build/doc/ $release_dir/doc
-
-  build/doc.sh release-index $release_dir/index.html
-
  tree -L 3 $release_root_dir
  
  ls -l $deploy_repo/releases.html
@@ -478,10 +579,15 @@ _releases-html-header() {
        color: #555;
        padding-left: 1em;
      }
+      #home-link {
+        text-align: right;
+      }
    </style>
  </head>
  <body>
-    <p></p>
+    <p id="home-link">
+      <a href="/">oilshell.org</a>
+    </p>
    <h1>Oil Releases</h1>
 EOF
 }
diff --git a/test/publish.sh b/test/publish.sh
@@ -47,22 +47,8 @@ unit() {
  echo 'Hello from publish.sh'
 }

-# Wild Tests.  NOTE: There's code for this in the oilshell.org repo.
-compress-wild() {
-  local dest=_tmp/wild-deploy
-  mkdir -p $dest
-
-  local out=$PWD/$dest/wild.wwz  # abs path for zip
-  rm -f -v $out
-
-  pushd _tmp/wild/www
-  time zip -r -q $out .  # recursive, quiet
-  popd
-
-  test/wild-runner.sh link-static $dest
-  ls -l -h $dest
-}
-
+# NOTE: Have to copy the web/ dir too?  For testing, use ./local.sh
+# test-release-tree.
 wild() {
  local user=$1
  local host=$2
@@ -72,8 +58,8 @@ wild() {

  ssh $user@$host mkdir -p $dest

-  rsync --archive --verbose --copy-links \
-    _tmp/wild-deploy/ $user@$host:$dest/
+  rsync --archive --verbose \
+    _release/VERSION/test/wild.wwz $user@$host:$dest/

  echo "Visit http://$dest/wild.wwz/"
 }
diff --git a/test/sh_spec.py b/test/sh_spec.py
@@ -644,7 +644,7 @@ def BeginCases(self, test_file):
 <!DOCTYPE html>
 <html>
  <head>
-    <link href="spec-tests.css" rel="stylesheet">
+    <link href="../../web/spec-tests.css" rel="stylesheet">
  </head>
  <body>
    <p id="home-link">