Change the structure of the compiler to use simple functions.

Remove a tremendous amount of object-oriented dispatch spaghetti.

There are now 4 bytecode differences shown by regtest.sh, but I think
they're of the same harmless variety as the util_opy.py case.  Have to
investigate!

opy/compiler2/misc.py

@@ -43,11 +43,3 @@ def mangle(name, klass):
        klass = klass[:MANGLE_LEN-tlen]

    return "_%s%s" % (klass, name)

def set_filename(filename, tree):
    """Set the filename attribute to filename on every node in tree"""
    worklist = [tree]
    while worklist:
        node = worklist.pop(0)
        node.filename = filename
        worklist.extend(node.getChildNodes())

opy/compiler2/pycodegen.py

@@ -3,7 +3,6 @@

from . import ast, syntax
from .visitor import walk
from .transformer import parse
from . import pyassem, misc, future, symbols
from .consts import (
    SC_LOCAL, SC_GLOBAL_IMPLICIT, SC_GLOBAL_EXPLICIT, SC_FREE, SC_CELL)
@@ -47,73 +46,37 @@ def getPycHeader(filename):
    return PY27_MAGIC + mtime


def compile(source, filename, mode, flags=None, dont_inherit=None, transformer=None):
def set_filename(filename, tree):
    """Set the filename attribute to filename on every node in tree"""
    worklist = [tree]
    while worklist:
        node = worklist.pop(0)
        node.filename = filename
        worklist.extend(node.getChildNodes())


def compile(parse_tree, filename, mode, flags=None, dont_inherit=None,
            transformer=None):
    """Replacement for builtin compile() function"""
    if flags is not None or dont_inherit is not None:
        raise RuntimeError, "not implemented yet"
        raise RuntimeError("not implemented yet")

    as_tree = transformer.transform(parse_tree)

    set_filename(filename, as_tree)
    syntax.check(as_tree)

    if mode == "single":
        gen = Interactive(source, filename)
        gen = InteractiveCodeGenerator(as_tree)
    elif mode == "exec":
        gen = Module(source, filename)
        gen = ModuleCodeGenerator(as_tree)
    elif mode == "eval":
        gen = Expression(source, filename)
        gen = ExpressionCodeGenerator(as_tree)
    else:
        raise ValueError("compile() 3rd arg must be 'exec' or "
                         "'eval' or 'single'")
    gen.compile(transformer=transformer)
    return gen.code

class AbstractCompileMode(object):

    mode = None # defined by subclass

    def __init__(self, source, filename):
        self.source = source
        self.filename = filename
        self.code = None

    def _get_tree(self, transformer=None):
        tree = parse(self.source, self.mode, transformer=transformer)
        misc.set_filename(self.filename, tree)
        syntax.check(tree)
        return tree

    def compile(self):
        pass # implemented by subclass

    def getCode(self):
        return self.code

class Expression(AbstractCompileMode):

    mode = "eval"

    def compile(self, transformer=None):
        tree = self._get_tree(transformer=transformer)
        gen = ExpressionCodeGenerator(tree)
        self.code = gen.getCode()

class Interactive(AbstractCompileMode):

    mode = "single"

    def compile(self, transformer=None):
        tree = self._get_tree(transformer=transformer)
        gen = InteractiveCodeGenerator(tree)
        self.code = gen.getCode()

class Module(AbstractCompileMode):

    mode = "exec"
    return gen.getCode()

    def compile(self, display=0, transformer=None):
        tree = self._get_tree(transformer=transformer)
        gen = ModuleCodeGenerator(tree)
        if display:
            import pprint
            print(pprint.pprint(tree))
        self.code = gen.getCode()

class LocalNameFinder(object):
    """Find local names in scope"""

opy/compiler2/transformer.py

@@ -33,7 +33,6 @@
from ..pytree import type_repr



symbol = None

def Init(sym):
@@ -167,26 +166,6 @@ class WalkerError(StandardError):
    pass


def parseFile(path):
    f = open(path, "U")
    # XXX The parser API tolerates files without a trailing newline,
    # but not strings without a trailing newline.  Always add an extra
    # newline to the file contents, since we're going through the string
    # version of the API.
    src = f.read() + "\n"
    f.close()
    return parse(src)

def parse(buf, mode="exec", transformer=None):
    tr = transformer or Transformer()
    if mode == "exec" or mode == "single":
        return tr.parsesuite(buf)
    elif mode == "eval":
        return tr.parseexpr(buf)
    else:
        raise ValueError("compile() arg 3 must be"
                         " 'exec' or 'eval' or 'single'")

def asList(nodes):
    l = []
    for item in nodes:
@@ -228,9 +207,6 @@ class Transformer(object):

    Exposes the following methods:
        tree = transform(ast_tree)
        tree = parsesuite(text)
        tree = parseexpr(text)
        tree = parsefile(fileob | filename)
    """

    def __init__(self):
@@ -256,22 +232,6 @@ def transform(self, tree):
        #    tree = parser.st2tuple(tree, line_info=1)
        return self.compile_node(tree)

    def parsesuite(self, text):
        """Return a modified parse tree for the given suite text."""
        raise AssertionError("Shouldn't use stdlib parser")
        #return self.transform(parser.suite(text))

    def parseexpr(self, text):
        """Return a modified parse tree for the given expression text."""
        raise AssertionError("Shouldn't use stdlib parser")
        #return self.transform(parser.expr(text))

    def parsefile(self, file):
        """Return a modified parse tree for the contents of the given file."""
        if type(file) == type(''):
            file = open(file)
        return self.parsesuite(file.read())

    # --------------------------------------------------------------
    #
    # PRIVATE METHODS
@@ -1557,23 +1517,6 @@ def get_docstring(self, node, n=None):
        return None


class Pgen2Transformer(Transformer):
  def __init__(self, py_parser, printer):
    Transformer.__init__(self)
    self.py_parser = py_parser
    self.printer = printer

  def parsesuite(self, text):
    tree = self.py_parser.suite(text)
    #self.printer.Print(tree)
    return self.transform(tree)

  def parseexpr(self, text):
    tree = self.py_parser.expr(text)
    #self.printer.Print(tree)
    return self.transform(tree)


def debug_tree(tree):
    l = []
    for elt in tree:

opy/opy_main.py

@@ -76,26 +76,6 @@ def WriteGrammar(grammar_path, pickle_path):
    log("Writing failed: %s", e)


# Emulate the interface that Transformer expects from parsermodule.c.
class Pgen2PythonParser(object):
  def __init__(self, driver, start_symbol):
    self.driver = driver
    self.start_symbol = start_symbol

  # TODO: Remove this redundancy
  def suite(self, text):
    f = cStringIO.StringIO(text)
    tokens = tokenize.generate_tokens(f.readline)
    tree = self.driver.parse_tokens(tokens, start_symbol=self.start_symbol)
    return tree

  def expr(self, text):
    f = cStringIO.StringIO(text)
    tokens = tokenize.generate_tokens(f.readline)
    tree = self.driver.parse_tokens(tokens, start_symbol=self.start_symbol)
    return tree


def CountTupleTree(tu):
  """Count the nodes in a tuple parse tree."""
  if isinstance(tu, tuple):
@@ -251,14 +231,12 @@ def OpyCommandMain(argv):
    py_path = argv[1]
    out_path = argv[2]

    py_parser = Pgen2PythonParser(dr, FILE_INPUT)
    printer = TupleTreePrinter(transformer._names)
    tr = transformer.Pgen2Transformer(py_parser, printer)

    # TODO: It shouldn't suck the whole file in!  It should read it line by line.
    with open(py_path) as f:
      contents = f.read()
    co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
      tokens = tokenize.generate_tokens(f.readline)
      parse_tree = dr.parse_tokens(tokens, start_symbol=FILE_INPUT)
      tr = transformer.Transformer()
      co = pycodegen.compile(parse_tree, py_path, 'exec', transformer=tr)
    log("Code length: %d", len(co.co_code))

    # Write the .pyc file
@@ -269,12 +247,11 @@ def OpyCommandMain(argv):

  elif action == 'eval':  # Like compile, but parses to a code object and prints it
    py_expr = argv[1]

    py_parser = Pgen2PythonParser(dr, gr.symbol2number['eval_input'])
    printer = TupleTreePrinter(transformer._names)
    tr = transformer.Pgen2Transformer(py_parser, printer)

    co = pycodegen.compile(py_expr, '<eval input>', 'eval', transformer=tr)
    f = cStringIO.StringIO(py_expr)
    tokens = tokenize.generate_tokens(f.readline)
    parse_tree = dr.parse_tokens(tokens, start_symbol=gr.symbol2number['eval_input'])
    tr = transformer.Transformer()
    co = pycodegen.compile(parse_tree, '<eval input>', 'eval', transformer=tr)

    v = dis_tool.Visitor()
    v.show_code(co)
@@ -285,11 +262,12 @@ def OpyCommandMain(argv):
  elif action == 'repl':  # Like eval in a loop
    while True:
      py_expr = raw_input('opy> ')
      py_parser = Pgen2PythonParser(dr, gr.symbol2number['eval_input'])
      printer = TupleTreePrinter(transformer._names)
      tr = transformer.Pgen2Transformer(py_parser, printer)

      co = pycodegen.compile(py_expr, '<eval input>', 'eval', transformer=tr)
      f = cStringIO.StringIO(py_expr)
      tokens = tokenize.generate_tokens(f.readline)
      # TODO: change this to 'single input'?  Why doesn't this work?
      parse_tree = dr.parse_tokens(tokens, start_symbol=gr.symbol2number['eval_input'])
      tr = transformer.Transformer()
      co = pycodegen.compile(parse_tree, '<REPL input>', 'single', transformer=tr)

      v = dis_tool.Visitor()
      v.show_code(co)
@@ -351,13 +329,14 @@ def OpyCommandMain(argv):
    opy_argv = argv[1:]

    if py_path.endswith('.py'):
      py_parser = Pgen2PythonParser(dr, FILE_INPUT)
      printer = TupleTreePrinter(transformer._names)
      tr = transformer.Pgen2Transformer(py_parser, printer)
      with open(py_path) as f:
        contents = f.read()
      co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
      #py_parser = Pgen2PythonParser(dr, FILE_INPUT)
      #printer = TupleTreePrinter(transformer._names)
      #tr = transformer.Pgen2Transformer(py_parser, printer)
      #with open(py_path) as f:
      #  contents = f.read()
      #co = pycodegen.compile(contents, py_path, 'exec', transformer=tr)
      #execfile.run_code_object(co, opy_argv)
      pass

    elif py_path.endswith('.pyc') or py_path.endswith('.opyc'):
      with open(py_path) as f: