Document the three OPy builds.

- Polish the OPy README
- Move the './build.sh oil-repo' output to '_tmp/repo-with-opy'.
- bin/osh-byterun is moved to opy/_tmp/repo-with-opy so we always use
  the OPy bytecode with byterun.  We don't care about CPython bytecode
  with byterun as much.

opy/README.md

@@ -43,51 +43,63 @@ Also:
Getting started
---------------

Start with https://github.com/oilshell/oil/wiki/Contributing .  This is
necessary to build the `py27.grammar` file and so forth.
Do the "Quick Start" in "in https://github.com/oilshell/oil/wiki/Contributing .
This is necessary to build the `py27.grammar` file and so forth.

Then:
After Oil is setup, we can try out OPy.  Run these commands (and let me know if
any of them doesn't work):

    opy$ ../bin/opyc run gold/hello_py2.py  # basic test of compiler and runtime

Compile Oil with the OPy compiler:

    $ ./build.sh oil-repo  # makes _tmp/osh-opy and _tmp/osh-ccompile
    $ ./build.sh oil-repo  # makes _tmp/repo-with-opy and _tmp/repo-with-cpython

Run Oil unit tests, compiled with OPy, under CPython:
Run Oil unit tests, compiled with OPy, under **CPython**:

    $ ./test.sh oil-unit

Run Oil unit tests, compiled with OPy, under byterun (OPyPy):
Run Oil unit tests, compiled with OPy, under **byterun**:

    $ ./test.sh oil-unit-byterun   # Run Oil unit tests, compiled with OPy, under CPython

Gold tests in gold/:
Gold tests in `gold/` compare the output of CPython vs. byterun:

    $ ./test.sh gold

Oil spec tests (slow):
Oil spec tests under byterun (slow):

    opy$ ./test.sh spec smoke  # like $REPO_ROOT/test/spec.sh smoke
    opy$ ./test.sh spec all    # like $REPO_ROOT/test/spec.sh all

They can be run manually like this:
FYI, they can be run manually like this:

    $ gold/regex_compile.py  # run with CPython
    $ ../bin/opyc run gold/regex_compile.py

Demo: speed difference between OSH under CPython and OSH under byterun:
Demo of the speed difference between OSH under CPython and OSH under byterun:

    ./demo.sh osh-byterun-speed

OPy Compiler Regtest
--------------------

This uses golden data in `_regtest/`.
This uses an old snapshot of the repo in `_regtest/`.

    ./regtest.sh compile  # note: different files than 'build.sh oil-repo'
    ./regtest.sh compile
    ./regtest.sh verify-golden

Notes on Three OPy Builds
-------------------------

- `$REPO_ROOT/_build/oil/bytecode-opy`: Bytecode for the release binary.  Built
  by `Makefile`.
- `$REPO_ROOT/opy/_tmp/repo-with-opy`: The entire repo with OPy.  For running
  Oil unit/spec tests under byterun, etc.  Built by `./build.sh oil-repo`.
- `$REPO_ROOT/opy/_tmp/regtest`: The snapshot of Python files in `opy/_regtest`
  are compiled, so we are insensitive to repo changes.  Built by `./regtest.sh
  compile`.

OPy Compiler Divergences from CPython
----------------------------

opy/build.sh

@@ -45,12 +45,8 @@ _compile-tree() {
    # TODO: Get rid of stdlib (compile) and compiler2.  Now that OPy works, we
    # just want opy and ccompile.

    if test $version = stdlib; then
      _stdlib-compile-one $src_tree/${rel_path} $dest
    elif test $version = compiler2; then
      _compile2-one $src_tree/${rel_path} $dest
    elif test $version = ccompile; then
      _ccompile-one $src_tree/${rel_path} $dest
    if test $version = ccompile; then
      misc/ccompile.py $src_tree/${rel_path} $dest
    elif test $version = opy; then
      $THIS_DIR/../bin/opyc compile $src_tree/${rel_path} $dest
    else
@@ -80,6 +76,7 @@ compile-manifest() {
  done
}

# UNUSED
make-mains() {
  local dir=${1:-_tmp/oil-opy}

@@ -105,7 +102,7 @@ make-mains() {
# Instead of printing .pyc, modify build/app_deps.py to print _tmp/oil/*.pyc !

_fill-oil-tree() {
  local dir=${1:-_tmp/oil-opy}
  local dir=${1:-_tmp/repo-with-opy}

  cp -v ../osh/{osh,types}.asdl $dir/osh
  cp -v ../core/runtime.asdl $dir/core
@@ -119,6 +116,14 @@ _fill-oil-tree() {
  # Running core/process_test.py depends on this existing!
  mkdir -v -p $dir/_tmp

  local stub=$dir/bin/osh-byterun
  cat >$stub <<'EOF'
#!/bin/bash
readonly THIS_DIR=$(cd $(dirname $0) && pwd)
exec python $THIS_DIR/opy_.pyc opyc run $THIS_DIR/oil.pyc osh "$@"
EOF
  chmod +x $stub

  #make-mains $dir
}

@@ -132,11 +137,11 @@ oil-repo() {
  local repo_root=$(cd $THIS_DIR/.. && pwd)
  local files=( $(oil-python-sources $repo_root) )  # array

  _compile-tree $repo_root _tmp/oil-ccompile/ ccompile "${files[@]}"
  _compile-tree $repo_root _tmp/oil-opy/ opy "${files[@]}"
  _compile-tree $repo_root _tmp/repo-with-cpython/ ccompile "${files[@]}"
  _compile-tree $repo_root _tmp/repo-with-opy/ opy "${files[@]}"

  _fill-oil-tree _tmp/oil-ccompile/ 
  _fill-oil-tree _tmp/oil-opy/
  _fill-oil-tree _tmp/repo-with-cpython
  _fill-oil-tree _tmp/repo-with-opy
}

_oil-bin-manifest() {

opy/common.sh

@@ -25,35 +25,6 @@ opy_() {
  PYTHONPATH=$THIS_DIR $THIS_DIR/../bin/opy_.py "$@"
}

_stdlib-compile-one() {
  # Run it from source, so we can patch.  Bug still appears.

  #$PY27/python misc/stdlib_compile.py "$@"

  # No with statement
  #~/src/Python-2.4.6/python misc/stdlib_compile.py "$@"

  # NOT here
  #~/src/Python-2.6.9/python misc/stdlib_compile.py "$@"

  # Bug appears in Python 2.7.9 too!
  #~/src/Python-2.7.9/python misc/stdlib_compile.py "$@"

  # Why is it in 2.7.2?  No hash randomization there?
  #~/src/Python-2.7.2/python misc/stdlib_compile.py "$@"

  # Woah it took 51 iterations to find!
  # Much rarer in Python 2.7.0.  100 iterations didn't find it?
  # Then 35 found it.  Wow.
  ~/src/Python-2.7/python misc/stdlib_compile.py "$@"

  #misc/stdlib_compile.py "$@"
}

_ccompile-one() {
  misc/ccompile.py "$@"
}

# NOTES:
# - Exclude _devbuild/cpython-full, but include _devbuild/gen.
# - must exclude opy/testdata/, because some of it can't be compiled

opy/demo.sh

@@ -9,14 +9,18 @@ set -o errexit

readonly THIS_DIR=$(cd $(dirname $0) && pwd)

osh-byterun() {
  opy/_tmp/repo-with-opy/bin/osh-byterun "$@"
}

# Show the difference between OSH running under CPython and OSH running under
# byterun.
osh-byterun-speed() {
  pushd $THIS_DIR/..

  local prog='for i in $(seq 10); do echo $i; done'
  time bin/osh -c "$prog"
  time bin/osh-byterun -c "$prog"
  time osh-byterun -c "$prog"

  popd
}
@@ -26,7 +30,7 @@ osh-byterun-parse() {

  pushd $THIS_DIR/..
  time bin/osh -n -c "$prog"
  time bin/osh-byterun -n -c "$prog"
  time osh-byterun -n -c "$prog"
  popd
}

opy/test.sh

@@ -32,7 +32,7 @@ oil-opy() {
#

oil-unit() {
  local dir=${1:-_tmp/oil-opy}
  local dir=${1:-_tmp/repo-with-opy}
  local vm=${2:-cpython}  # byterun or cpython

  pushd $dir
@@ -162,11 +162,6 @@ re-dis() {
  ../bin/opyc dis /usr/lib/python2.7/sre_parse.pyc
}


unit() {
  PYTHONPATH=. "$@"
}

# Spec tests under byterun.
spec() { 
  local action=$1  # e.g. 'smoke' or 'all'
@@ -180,14 +175,7 @@ spec() {

  # HACK to get around __import__ problem with byterun.

  local stub=opy/_tmp/oil-opy/bin/osh-byterun 
  cat >$stub <<'EOF'
#!/bin/bash
readonly THIS_DIR=$(cd $(dirname $0) && pwd)
exec python $THIS_DIR/opy_.pyc opyc run $THIS_DIR/oil.pyc osh "$@"
EOF
  chmod +x $stub

  local stub=opy/_tmp/repo-with-opy/bin/osh-byterun 
  OSH_OVM=$stub test/spec.sh $action "$@"
  popd
}