Full OPyPy chain is now working!

- opy_main.py compile now uses pgen2 + compiler2.  I reapplied the fixes
  from compiler/ (to be deleted).  For example:
  - 'symbol' module replacement.  The compiler2 module depended on the
    'symbol' stdlib module.  But we want to get that information from
    the grammar, as extracted by pgen2.
  - make sure that 'import parser' is never used.
- Change bin/opypy-osh to point at the the opy-compiled version in
  '_tmp/osh-opy'

Test results:

- Unit tests passing on _tmp/osh-opy
- Spec tests passing with bin/opypy-osh
  - Change loop test to be more robust toward .swp files

bin/opypy-osh

@@ -5,5 +5,6 @@
readonly THIS_DIR=$(cd $(dirname $0) && pwd)
readonly OIL_DIR=$THIS_DIR/..

export PYTHONPATH=$OIL_DIR/opy/_tmp/compile2 
$OIL_DIR/opy/byterun/__main__.py -c $OIL_DIR/opy/_tmp/osh-compile2/bin/oil.pyc osh "$@"
# Don't need PYTHONPATH because oil.py sets it
#export PYTHONPATH=$OIL_DIR/opy/_tmp/osh-opy
$OIL_DIR/opy/byterun/__main__.py -c $OIL_DIR/opy/_tmp/osh-opy/bin/oil.pyc osh "$@"

opy/common.sh

@@ -19,11 +19,16 @@ _compile-one() {
  # Python 2.7 doesn't have it.
  #local g=2to3.grammar 
  local g=py27.grammar
  PYTHONPATH=. ./opy_main.py $g old-compile "$@"
  # pgen2 + old compiler/
  #PYTHONPATH=. ./opy_main.py $g old-compile "$@"

  # opy is pgen2 + compiler2
  PYTHONPATH=. ./opy_main.py $g compile "$@"
}

_compile2-one() {
  local g=py27.grammar
  # The unmodified compiler2
  PYTHONPATH=. ./opy_main.py $g compile2 "$@"
}

opy/compiler2/__init__.py

@@ -28,4 +28,4 @@

from compiler2.transformer import parse, parseFile
from compiler2.visitor import walk
from compiler2.pycodegen import compile, compileFile
from compiler2.pycodegen import compile

opy/compiler2/pycodegen.py

@@ -34,21 +34,25 @@
TRY_FINALLY = 3
END_FINALLY = 4

def compileFile(filename, display=0):
    f = open(filename, 'U')
    buf = f.read()
    f.close()
    mod = Module(buf, filename)
    try:
        mod.compile(display)
    except SyntaxError:
        raise
    else:
        f = open(filename + "c", "wb")
        mod.dump(f)
        f.close()

def compile(source, filename, mode, flags=None, dont_inherit=None):
PY27_MAGIC = b'\x03\xf3\r\n'  # removed host dep imp.get_magic()

def getPycHeader(filename):
    # compile.c uses marshal to write a long directly, with
    # calling the interface that would also generate a 1-byte code
    # to indicate the type of the value.  simplest way to get the
    # same effect is to call marshal and then skip the code.
    mtime = os.path.getmtime(filename)
    mtime = struct.pack('<i', int(mtime))

    # Update for Python 3:
    # https://nedbatchelder.com/blog/200804/the_structure_of_pyc_files.html
    # https://gist.github.com/anonymous/35c08092a6eb70cdd723

    return PY27_MAGIC + mtime


def compile(source, filename, mode, flags=None, dont_inherit=None, transformer=None):
    """Replacement for builtin compile() function"""
    if flags is not None or dont_inherit is not None:
        raise RuntimeError, "not implemented yet"
@@ -62,7 +66,7 @@ def compile(source, filename, mode, flags=None, dont_inherit=None):
    else:
        raise ValueError("compile() 3rd arg must be 'exec' or "
                         "'eval' or 'single'")
    gen.compile()
    gen.compile(transformer=transformer)
    return gen.code

class AbstractCompileMode:
@@ -74,8 +78,8 @@ def __init__(self, source, filename):
        self.filename = filename
        self.code = None

    def _get_tree(self):
        tree = parse(self.source, self.mode)
    def _get_tree(self, transformer=None):
        tree = parse(self.source, self.mode, transformer=transformer)
        misc.set_filename(self.filename, tree)
        syntax.check(tree)
        return tree
@@ -108,29 +112,14 @@ class Module(AbstractCompileMode):

    mode = "exec"

    def compile(self, display=0):
        tree = self._get_tree()
    def compile(self, display=0, transformer=None):
        tree = self._get_tree(transformer=transformer)
        gen = ModuleCodeGenerator(tree)
        if display:
            import pprint
            print pprint.pprint(tree)
        self.code = gen.getCode()

    def dump(self, f):
        f.write(self.getPycHeader())
        marshal.dump(self.code, f)

    MAGIC = imp.get_magic()

    def getPycHeader(self):
        # compile.c uses marshal to write a long directly, with
        # calling the interface that would also generate a 1-byte code
        # to indicate the type of the value.  simplest way to get the
        # same effect is to call marshal and then skip the code.
        mtime = os.path.getmtime(self.filename)
        mtime = struct.pack('<i', mtime)
        return self.MAGIC + mtime

class LocalNameFinder:
    """Find local names in scope"""
    def __init__(self, names=()):
@@ -1554,7 +1543,3 @@ class AugSubscript(Delegator):

def wrap_aug(node):
    return wrapper[node.__class__](node)

if __name__ == "__main__":
    for file in sys.argv[1:]:
        compileFile(file)