Running all spec tests with _tmp/oil-opy/bin/osh.

Move shell functions that work to {build,test}.sh, and document them in README.md. These scripts are Analogous to the oil/{build,test}/ directories. Make shell script stubs for _tmp/oil-opy. osh --help and osh --version now work.
oilshell · Feb 24, 2018 · 6fb35c8a6869a6577cdc86a42e5a44389221543e · 6fb35c8
1 parent 37df248
commit 6fb35c8a6869a6577cdc86a42e5a44389221543e
Unified Split

Showing with 226 additions and 155 deletions.

+4 −0 build/common.sh

+11 −5 opy/README.md

+125 −0 opy/build.sh

+0 −135 opy/smoke.sh

+76 −0 opy/test.sh

+4 −11 scripts/run.sh

+6 −4 test/spec.sh
diff --git a/build/common.sh b/build/common.sh
@@ -18,6 +18,10 @@ readonly PY27=Python-2.7.13

 readonly PREPARE_DIR=_devbuild/cpython-full

+# Used by scripts/run.sh and opy/build.sh
+readonly OIL_SYMLINKS=(oil oilc osh sh wok boil true false)
+
+
 log() {
  echo "$@" >&2
 }
diff --git a/opy/README.md b/opy/README.md
@@ -10,16 +10,22 @@ Getting started / smoke test:

 Compiling Oil:

-    ./smoke.sh compile-osh-tree   # makes _tmp/osh-opy and _tmp/osh-ccompile
-    ./smoke.sh test-unit  # Run Oil unit tests
+    ./build.sh oil-repo  # makes _tmp/osh-opy and _tmp/osh-ccompile

-TODO:
+Testing:

-Move important stuff to build.sh.  smoke.sh doesn't make sense.
+    ./test.sh unit  # Run Oil unit tests
+
+Test the binary:
+
+    ./test.sh osh-help 
+    ./test.sh osh-version 
+    ./test.sh spec smoke
+    ./test.sh spec all  # Failures due to $0

 Notes
 -----

 OSH tests don't run under byterun.  I probably don't care.

-    ./smoke.sh test-osh-tree '' byterun
+    ./test.sh unit '' byterun
diff --git a/opy/build.sh b/opy/build.sh
@@ -1,5 +1,10 @@
 #!/usr/bin/env bash
 #
+# Compile Python code with OPy.
+#
+# NOTE: this is roughly analogous to build/actions.sh and may be moved
+# there.
+#
 # Usage:
 #   ./build.sh <function name>

@@ -8,10 +13,130 @@ set -o pipefail
 set -o errexit

 source common.sh
+source ../build/common.sh  # for OIL_SYMLINKS

 grammar() {
  mkdir -p _tmp
  opy_ pgen2 py27.grammar $GRAMMAR
 }

+md5-manifest() {
+  local tree=$1
+  pushd $tree
+  # size and name
+  find . -type f | sort | xargs stat --format '%s %n' | tee SIZES.txt
+  find . -type f | sort | xargs md5sum | tee MD5.txt
+  popd
+}
+
+_compile-tree() {
+  local src_tree=$1
+  local dest_tree=$2
+  local version=$3
+  shift 3
+
+  rm -r -f $dest_tree
+
+  #local ext=opyc
+  local ext=pyc
+
+  for rel_path in "$@"; do
+    echo $rel_path
+    local dest=${dest_tree}/${rel_path%.py}.${ext}
+    mkdir -p $(dirname $dest)
+
+    if test $version = stdlib; then
+      _stdlib-compile-one $src_tree/${rel_path} $dest
+    elif test $version = compiler2; then
+      _compile2-one $src_tree/${rel_path} $dest
+    elif test $version = ccompile; then
+      _ccompile-one $src_tree/${rel_path} $dest
+    elif test $version = opy; then
+      _compile-one $src_tree/${rel_path} $dest
+    else
+      die "bad"
+    fi
+  done
+
+  tree $dest_tree
+  md5-manifest $dest_tree
+}
+
+make-mains() {
+  local dir=${1:-_tmp/oil-opy}
+
+  # Have to use shells cripts rather than symlinks because .pyc files aren't
+  # executable.
+  # TODO: Use oil.ovm instead of system Python.
+  for link in "${OIL_SYMLINKS[@]}"; do
+    { echo '#!/bin/sh'
+      echo "main=$link"
+      echo 'exec python $(dirname $0)/oil.pyc $main "$@"'
+    } >$dir/bin/$link
+
+    chmod --verbose +x $dir/bin/$link
+  done
+}
+
+# TODO: Consolidate with the normal build.  This is like
+# build/oil-manifest.txt.  The Makefile rule for _build/oil/bytecode.zip calls
+# actions.sh files-manifest.
+#
+# Instead of printing .pyc, modify build/app_deps.py to print _tmp/oil/*.pyc !
+#
+# And then write a script to run spec tests against the oil-opy.ovm debug
+# oil1.ovm
+# oil1.ovm-dbg
+#
+# oil-with-cpython.ovm
+# oil-with-cpython.ovm-dbg
+
+_fill-oil-tree() {
+  local dir=${1:-_tmp/oil-opy}
+
+  cp -v ../osh/{osh,types}.asdl $dir/osh
+  cp -v ../core/runtime.asdl $dir/core
+  cp -v ../asdl/arith.asdl $dir/asdl
+  ln -v -s -f $PWD/../{libc,fastlex}.so $dir
+  ln -v -s -f $PWD/../oil-version.txt $dir
+
+  # Needed for help text.
+  ln -v -s -f --no-target-directory $PWD/../_build $dir/_build
+
+  make-mains $dir
+}
+
+# Compile with both compile() and OPy.
+# TODO:
+# - What about the standard library?  The whole app bundle should be
+# compiled with OPy.
+# - This could be part of the Travis build.  It will ensure no Python 2
+# print statements sneak in.
+oil-repo() {
+  local src=$(cd .. && echo $PWD)
+
+  # NOTE: Exclude _devbuild/cpython-full, but include _devbuild/gen.
+  local files=( $(find $src \
+              -name _tmp -a -prune -o \
+              -name _chroot -a -prune -o \
+              -name cpython-full -a -prune -o \
+              -name _deps -a -prune -o \
+              -name Python-2.7.13 -a -prune -o \
+              -name opy -a -prune -o \
+              -name 'test' -a -prune -o \
+              -name '*.py' -a -printf '%P\n') )
+
+  _compile-tree $src _tmp/oil-ccompile/ ccompile "${files[@]}"
+  _compile-tree $src _tmp/oil-opy/ opy "${files[@]}"
+
+  _fill-oil-tree _tmp/oil-ccompile/ 
+  _fill-oil-tree _tmp/oil-opy/
+
+  #_compile-tree $src _tmp/osh-compile2/ compiler2 "${files[@]}"
+
+  # Not deterministic!
+  #_compile-tree $src _tmp/osh-compile2.gold/ compiler2 "${files[@]}"
+  #_compile-tree $src _tmp/osh-stdlib/ stdlib "${files[@]}"
+}
+
 "$@"
diff --git a/opy/smoke.sh b/opy/smoke.sh
@@ -12,48 +12,6 @@ set -o errexit
 source common.sh
 source compare.sh  # for DIFF

-md5-manifest() {
-  local tree=$1
-  pushd $tree
-  # size and name
-  find . -type f | sort | xargs stat --format '%s %n' | tee SIZES.txt
-  find . -type f | sort | xargs md5sum | tee MD5.txt
-  popd
-}
-
-_compile-tree() {
-  local src_tree=$1
-  local dest_tree=$2
-  local version=$3
-  shift 3
-
-  rm -r -f $dest_tree
-
-  #local ext=opyc
-  local ext=pyc
-
-  for rel_path in "$@"; do
-    echo $rel_path
-    local dest=${dest_tree}/${rel_path%.py}.${ext}
-    mkdir -p $(dirname $dest)
-
-    if test $version = stdlib; then
-      _stdlib-compile-one $src_tree/${rel_path} $dest
-    elif test $version = compiler2; then
-      _compile2-one $src_tree/${rel_path} $dest
-    elif test $version = ccompile; then
-      _ccompile-one $src_tree/${rel_path} $dest
-    elif test $version = opy; then
-      _compile-one $src_tree/${rel_path} $dest
-    else
-      die "bad"
-    fi
-  done
-
-  tree $dest_tree
-  md5-manifest $dest_tree
-}
-
 _fill-opy-tree() {
  echo TODO: grammar pickle
 }
@@ -73,71 +31,6 @@ compile-opy-tree() {
  _compile-tree $src _tmp/opy-opy/ opy "${files[@]}"
 }

-make-links() {
-  local dir=${1:-_tmp/osh-opy}
-
-  local run=$PWD/../scripts/run.sh 
-
-  pushd $dir
-  $run make-pyc-links
-  popd
-  #tree $dir
-}
-
-_fill-osh-tree() {
-  local dir=${1:-_tmp/osh-opy}
-  cp -v ../osh/{osh,types}.asdl $dir/osh
-  cp -v ../core/runtime.asdl $dir/core
-  cp -v ../asdl/arith.asdl $dir/asdl
-  ln -v -s -f $PWD/../{libc,fastlex}.so $dir
-
-  # Needed for help text.
-  ln -v -s -f --no-target-directory $PWD/../_build $dir/_build
-
-  make-links $dir
-}
-
-osh-opy() {
-  python _tmp/osh-opy/bin/osh "$@"
-}
-
-test-help() {
-  osh-opy --help
-}
-
-# Compile with both compile() and OPy.
-# TODO:
-# - What about the standard library?  The whole app bundle should be
-# compiled with OPy.
-# - This could be part of the Travis build.  It will ensure no Python 2
-# print statements sneak in.
-compile-osh-tree() {
-  local src=$(cd .. && echo $PWD)
-
-  # NOTE: Exclude _devbuild/cpython-full, but include _devbuild/gen.
-  local files=( $(find $src \
-              -name _tmp -a -prune -o \
-              -name _chroot -a -prune -o \
-              -name cpython-full -a -prune -o \
-              -name _deps -a -prune -o \
-              -name Python-2.7.13 -a -prune -o \
-              -name opy -a -prune -o \
-              -name 'test' -a -prune -o \
-              -name '*.py' -a -printf '%P\n') )
-
-  _compile-tree $src _tmp/osh-ccompile/ ccompile "${files[@]}"
-  _compile-tree $src _tmp/osh-opy/ opy "${files[@]}"
-
-  _fill-osh-tree _tmp/osh-ccompile/ 
-  _fill-osh-tree _tmp/osh-opy/
-
-  #_compile-tree $src _tmp/osh-compile2/ compiler2 "${files[@]}"
-
-  # Not deterministic!
-  #_compile-tree $src _tmp/osh-compile2.gold/ compiler2 "${files[@]}"
-  #_compile-tree $src _tmp/osh-stdlib/ stdlib "${files[@]}"
-}
-
 zip-oil-tree() {
  #pushd _tmp/osh-opy
  rm -f _tmp/oil.zip
@@ -146,34 +39,6 @@ zip-oil-tree() {
  popd
 }

-# TODO:
-# - Run with oil.ovm{,-dbg}
-test-unit() {
-  local dir=${1:-_tmp/osh-opy}
-  local vm=${2:-cpython}  # byterun or cpython
-
-  pushd $dir
-  mkdir -p _tmp
-  #for t in {build,test,native,asdl,core,osh,test,tools}/*_test.py; do
-  for t in {asdl,core,osh}/*_test.pyc; do
-
-    echo $t
-    if test $vm = byterun; then
-      PYTHONPATH=. opy_ run $t
-    elif test $vm = cpython; then
-      PYTHONPATH=. python $t
-    else
-      die "Invalid VM $vm"
-    fi
-  done
-  popd
-}
-
-test-osh-smoke() {
-  local dir=${1:-_tmp/osh-opy}
-  local vm=${2:-cpython}  # byterun or cpython
-}
-
 write-speed() {
  cat >_tmp/speed.py <<EOF
 def do_sum(n):